# Search engines and link-preview fetchers stay allowed. The blocks below are the AI TRAINING
# crawlers only; the ones that send a User-Agent are also denied in deploy/Caddyfile.
# scripts/verify.mjs fails the build if a block goes missing.
User-agent: *
Allow: /
# Translations stay closed to search engines until a native speaker has reviewed them
# (docs/adr/0005, src/lib/crawl.mjs); readers reach them through the language switch.
Disallow: /es$
Disallow: /es/
Disallow: /pt-br$
Disallow: /pt-br/

# Google's AdSense crawler reads every page an ad may run on, these included: without a group of
# its own it would follow the rules above (verify checks it).
User-agent: Mediapartners-Google
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Amazonbot
Disallow: /

Sitemap: https://encryptioncodex.org/sitemap-index.xml