# crystal-ball.org — index the pages, cite them, leave the data API alone.
#
# Quoting or summarizing this site, in a search result or an AI answer? Credit
# CrystalBall next to what you use and link the page it came from
# (https://crystal-ball.org/llms.txt, "Attribution").
#
# Three groups, most specific first (a crawler obeys the first group that
# names it; anything unnamed falls through to "*"):
#   1. AI SEARCH ENGINES — every page is open: a search result that cites and
#      links here is the traffic we want.
#   2. AI TRAINING crawlers, bulk data collectors, SEO scrapers and AI CHAT
#      AGENTS (the fetchers a chat assistant sends for a person) — nothing.
#   3. Everyone else (Googlebot, Bingbot, Yeti/Naver, Applebot, DuckDuckBot,
#      link previews) — pages yes, the data API and the schema no.
# Google-Extended / Applebot-Extended / Webzio-Extended are robots-only tokens
# (they never fetch) that switch off TRAINING use of what the ordinary crawler
# fetched; Google Search, AI Overviews and Bing Copilot are unaffected.
# The Content-Signal line (contentsignals.org) says the same thing to any
# crawler that reads it: search yes, AI answers yes, AI training no.
# The data API is session-bound and not for automated access (/terms).
# /llms.txt describes the site for AI readers.

# 1. AI search engines
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow: /api/
Disallow: /docs
Disallow: /openapi.json

# 2. AI training / data collectors / SEO scrapers / AI chat agents: no access at all
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: YouBot
User-agent: meta-externalfetcher
User-agent: Amzn-User
User-agent: Google-Agent
User-agent: kagi-fetcher
User-agent: PhindBot
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Google-CloudVertexBot
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: TikTokSpider
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: AI2Bot
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: Omgilibot
User-agent: omgili
User-agent: Webzio-Extended
User-agent: Timpibot
User-agent: PanguBot
User-agent: PetalBot
User-agent: DeepSeekBot
User-agent: Kimi-User
User-agent: Kimi-Agent
User-agent: Kimi-SearchBot
User-agent: KimiBot
User-agent: DoubaoBot
User-agent: TongyiBot
User-agent: QwenBot
User-agent: ERNIEBot
User-agent: YiyanBot
User-agent: ChatGLM-Spider
User-agent: Manus-User
User-agent: Trae
User-agent: Baiduspider
User-agent: Baiduspider-image
User-agent: Baiduspider-render
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSeoBot
Disallow: /

# 3. Everyone else (Googlebot, Bingbot, Yeti, Applebot, previews…)
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow: /api/
Disallow: /api/players
Disallow: /api/v1/
Disallow: /api/v2/
Disallow: /api/export/
Disallow: /api/internal/
Disallow: /api/graphql
Disallow: /docs
Disallow: /openapi.json

Sitemap: https://crystal-ball.org/sitemap.xml