#
# Big shoutout to the humans reading this.
#

User-agent: ia_archiver
Disallow: /about

# Amazon - https://developer.amazon.com/amazonbot
User-agent: Amazonbot
Disallow: /

# Apple - https://support.apple.com/en-us/119829
User-agent: Applebot-Extended
Disallow: /

# ByteDance - not really documented, but known to cause a lot of load on servers
User-agent: Bytespider
Disallow: /

# Common Crawl - https://commoncrawl.org/ccbot
User-agent: CCBot
Disallow: /

# Anthropic - https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler
User-agent: ClaudeBot
Disallow: /

# Diffbot - https://docs.diffbot.com/reference/crawl-introduction
User-agent: Diffbot
Disallow: /

# Meta - https://developers.facebook.com/docs/sharing/bot/
User-agent: FacebookBot
Disallow: /

# OpenAI - https://platform.openai.com/docs/gptbot
User-agent: GPTBot
Disallow: /

# Google - https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers?hl=en#google-extended
User-agent: Google-Extended
Disallow: /

# Hive - https://imagesift.com/about
User-agent: ImagesiftBot
Disallow: /

# webz.io - https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/
User-agent: Omgili
Disallow: /

# webz.io - https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/
User-agent: OmgiliBot
Disallow: /

# Perplexity - https://docs.perplexity.ai/docs/perplexitybot
User-agent: PerplexityBot
Disallow: /

# Anthropic - undocumented but appears to still do something
User-agent: anthropic-ai
Disallow: /

# cohere - undocumented, but widely refrenced
User-agent: cohere-ai
Disallow: /

# Generated using https://codeberg.org/famfo/ai.txt.