# AfroMythos robots.txt (content-protection pack, 15 Oct 2026)
#
# AfroMythos is free to read. Search engines are welcome, and so are the
# assistants that fetch a page because a person asked about it and then link
# back here. What we turn away are the crawlers that collect the web to TRAIN
# AI models or to build datasets that are sold on.
# Terms: https://afromythos.com/copyright
#
# robots.txt is a request, not a lock: well-behaved crawlers follow it.
# Every page also carries <meta name="robots" content="noai, noimageai"> and
# the X-Robots-Tag: noai, noimageai header.

# --- Everyone else, including Googlebot, Bingbot, DuckDuckBot, Applebot,
# --- YandexBot, Baiduspider, PetalBot (Huawei Petal Search), Amzn-SearchBot,
# --- and search/answer bots that cite and link (OAI-SearchBot, Claude-SearchBot,
# --- PerplexityBot, DuckAssistBot, meta-webindexer), plus user-initiated
# --- fetchers (ChatGPT-User, Claude-User, Perplexity-User, meta-externalfetcher,
# --- Amzn-User) and link-preview bots (facebookexternalhit for WhatsApp/Facebook
# --- cards, Twitterbot, LinkedInBot, Slackbot): allowed.
User-agent: *
Allow: /

# --- AI training and dataset crawlers: not allowed ---

# OpenAI: GPTBot collects content that may train OpenAI's foundation models.
# (OAI-SearchBot = ChatGPT search results and ChatGPT-User = a person's request stay allowed.)
User-agent: GPTBot
Disallow: /

# Anthropic: ClaudeBot gathers content that may train Claude models; anthropic-ai
# and Claude-Web are older Anthropic tokens. (Claude-SearchBot and Claude-User stay allowed.)
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
Disallow: /

# Common Crawl: a public web archive that is one of the largest sources of AI training data.
User-agent: CCBot
Disallow: /

# Google-Extended is not a crawler but a control token: it opts content out of
# training Gemini models (and Gemini grounding). It does not affect Google Search.
User-agent: Google-Extended
Disallow: /

# Applebot-Extended opts content out of training Apple's foundation models.
# Applebot itself (Siri, Spotlight, Safari search) stays allowed.
User-agent: Applebot-Extended
Disallow: /

# ByteDance (TikTok): Bytespider and TikTokSpider collect LLM training data.
User-agent: Bytespider
User-agent: TikTokSpider
Disallow: /

# Meta: meta-externalagent crawls for training Meta's AI models; FacebookBot is
# Meta's older training crawler. (facebookexternalhit, which draws the preview
# card when a link is shared on WhatsApp, Facebook or Instagram, stays allowed.)
User-agent: meta-externalagent
User-agent: FacebookBot
Disallow: /

# Amazon: Amazonbot "may be used to train Amazon AI models".
# (Amzn-SearchBot, which Amazon says does not crawl for AI training, and Amzn-User stay allowed.)
User-agent: Amazonbot
Disallow: /

# Cohere: cohere-training-data-crawler collects training data for Cohere's models;
# cohere-ai is Cohere's older token.
User-agent: cohere-training-data-crawler
User-agent: cohere-ai
Disallow: /

# Allen Institute for AI: AI2Bot (and Ai2Bot-Dolma) collect text to train open language models.
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
Disallow: /

# Huawei PanguBot collects training data for the PanGu models.
# (PetalBot, Huawei's search crawler, stays allowed.)
User-agent: PanguBot
Disallow: /

# Data brokers whose crawls are sold or used to build AI knowledge graphs and training sets.
User-agent: Diffbot
User-agent: omgili
User-agent: Omgilibot
User-agent: Webzio-Extended
User-agent: Timpibot
User-agent: ImagesiftBot
User-agent: Kangaroo Bot
Disallow: /

# img2dataset downloads image collections for AI training (it also honours the
# noai / noimageai headers this site sends).
User-agent: img2dataset
Disallow: /

Sitemap: https://afromythos.com/sitemap.xml