# Econ3x3 crawler policy (production / www only — worker/index.ts serves its own
# allow-all + noindex robots.txt on every other host).
#
# Intent: DO NOT train on our articles. DO retrieve, index and cite them, so
# that AI assistants can link readers back to econ3x3.org.
#
# These are separable because the major vendors run SEPARATE crawlers for the
# two jobs. Blocking a training crawler does not remove us from that
# assistant's search index. Caveat: robots.txt is honour-system — obeyed by the
# vendors below, not enforced. Do NOT switch on Cloudflare's blanket "Block AI
# bots" rule: it is all-or-nothing and would take the retrieval crawlers down
# with the training ones.

# --- Default: crawl freely (Googlebot, Bingbot, Scholar, everything else) ---
User-agent: *
Allow: /

# --- Training crawlers: blocked ---
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

# --- Retrieval / citation crawlers: explicitly welcome ---
# These are what put econ3x3.org links inside AI answers. Listed explicitly
# rather than left to the wildcard so the intent survives future edits: a
# crawler obeys only its own most-specific group, never a merge of its group
# and "*". Do not delete these lines assuming they are redundant.

# ChatGPT's search index, and live fetches when a user asks about us
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Claude's search index and user-initiated fetches
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google-Extended is DELIBERATELY not blocked. It is the one crawler that
# couples both sides: it gates Gemini/Vertex training AND Gemini grounding, so
# blocking it would also cost us citations in Google's AI surfaces — by far the
# largest reach we have. Revisit if that coupling ever changes.

Sitemap: https://www.econ3x3.org/sitemap-index.xml