{
  "checked": "2026-11-10",
  "method": "UA token claim plus agent-specific IPv4/IPv6 snapshot membership",
  "agents": [
    {
      "operator": "OpenAI",
      "token": "GPTBot",
      "purpose": "Training",
      "docKey": "openai",
      "doc": "https://developers.openai.com/api/docs/bots",
      "policy": "Yes; independent training control",
      "ipKey": "ip_gpt",
      "ipUrl": "https://openai.com/gptbot.json",
      "kind": "agent",
      "robots": "GPTBot",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "OpenAI",
      "token": "OAI-SearchBot",
      "purpose": "Search index",
      "docKey": "openai",
      "doc": "https://developers.openai.com/api/docs/bots",
      "policy": "Yes; independent search control",
      "ipKey": "ip_search",
      "ipUrl": "https://openai.com/searchbot.json",
      "kind": "agent",
      "robots": "OAI-SearchBot",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "OpenAI",
      "token": "ChatGPT-User",
      "purpose": "User-triggered fetch",
      "docKey": "openai",
      "doc": "https://developers.openai.com/api/docs/bots",
      "policy": "\u201crobots.txt rules may not apply\u201d",
      "ipKey": "ip_chatgpt",
      "ipUrl": "https://openai.com/chatgpt-user.json",
      "kind": "agent",
      "robots": "ChatGPT-User",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "Anthropic",
      "token": "ClaudeBot",
      "purpose": "Training",
      "docKey": "anthropic",
      "doc": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "policy": "Yes; \u201chonoring industry standard directives in robots.txt\u201d",
      "ipKey": "ip_claude",
      "ipUrl": "https://claude.com/crawling/bots.json",
      "kind": "agent",
      "robots": "ClaudeBot",
      "note": "Shared Anthropic list verifies operator, not the precise bot role.",
      "checked": "2026-11-10"
    },
    {
      "operator": "Anthropic",
      "token": "Claude-SearchBot",
      "purpose": "Search index",
      "docKey": "anthropic",
      "doc": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "policy": "Yes; \u201chonoring industry standard directives in robots.txt\u201d",
      "ipKey": "ip_claude",
      "ipUrl": "https://claude.com/crawling/bots.json",
      "kind": "agent",
      "robots": "Claude-SearchBot",
      "note": "Shared list; see the Anthropic verification method.",
      "checked": "2026-11-10"
    },
    {
      "operator": "Anthropic",
      "token": "Claude-User",
      "purpose": "User-triggered fetch",
      "docKey": "anthropic",
      "doc": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "policy": "Yes; \u201chonoring industry standard directives in robots.txt\u201d",
      "ipKey": "ip_claude",
      "ipUrl": "https://claude.com/crawling/bots.json",
      "kind": "agent",
      "robots": "Claude-User",
      "note": "Shared list; see the Anthropic verification method.",
      "checked": "2026-11-10"
    },
    {
      "operator": "Perplexity",
      "token": "PerplexityBot",
      "purpose": "Search index",
      "docKey": "perplexity",
      "doc": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "policy": "Yes; independent robots control",
      "ipKey": "ip_perplexity",
      "ipUrl": "https://www.perplexity.ai/perplexitybot.json",
      "kind": "agent",
      "robots": "PerplexityBot",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "Perplexity",
      "token": "Perplexity-User",
      "purpose": "User-triggered fetch",
      "docKey": "perplexity",
      "doc": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "policy": "\u201cgenerally ignores robots.txt rules\u201d",
      "ipKey": "ip_perplexity_user",
      "ipUrl": "https://www.perplexity.ai/perplexity-user.json",
      "kind": "agent",
      "robots": "Perplexity-User",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "Google",
      "token": "Google-Extended",
      "purpose": "Training and Gemini grounding policy",
      "docKey": "google",
      "doc": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "policy": "Policy token; \u201ccontrol capacity\u201d",
      "ipKey": "",
      "ipUrl": "",
      "kind": "policy",
      "robots": "Google-Extended",
      "note": "No HTTP user agent. Does not control Google Search inclusion or ranking; Gemini grounding is also covered, so this is not training-only.",
      "checked": "2026-11-10"
    },
    {
      "operator": "Google",
      "token": "Googlebot",
      "purpose": "Search index, including AI Search features",
      "docKey": "google",
      "doc": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "policy": "Yes; \u201calways obey robots.txt rules\u201d",
      "ipKey": "ip_google_common",
      "ipUrl": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
      "kind": "agent",
      "robots": "Googlebot",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "Apple",
      "token": "Applebot",
      "purpose": "Search; data may also train models",
      "docKey": "apple",
      "doc": "https://support.apple.com/en-us/119829",
      "policy": "Yes; \u201crespects standard robots.txt directives\u201d",
      "ipKey": "ip_apple",
      "ipUrl": "https://search.developer.apple.com/applebot.json",
      "kind": "agent",
      "robots": "Applebot",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "Apple",
      "token": "Applebot-Extended",
      "purpose": "Training-use policy",
      "docKey": "apple",
      "doc": "https://support.apple.com/en-us/119829",
      "policy": "Policy token; opt out of training",
      "ipKey": "",
      "ipUrl": "",
      "kind": "policy",
      "robots": "Applebot-Extended",
      "note": "No separate crawler. Allowing Applebot while disallowing this token retains search crawling.",
      "checked": "2026-11-10"
    },
    {
      "operator": "Microsoft",
      "token": "bingbot",
      "purpose": "Bing index; supports Copilot grounding",
      "docKey": "bing_doc",
      "doc": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "policy": "Yes; \u201cBingbot honors robots.txt directives\u201d",
      "ipKey": "ip_bing",
      "ipUrl": "https://www.bing.com/toolbox/bingbot.json",
      "kind": "agent",
      "robots": "bingbot",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "Meta",
      "token": "meta-externalagent",
      "purpose": "Training and product indexing",
      "docKey": "meta",
      "doc": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "policy": "Uses \u201cDisallow\u201d (operator example)",
      "ipKey": "",
      "ipUrl": "",
      "kind": "agent",
      "robots": "meta-externalagent",
      "note": "Official documentation was served in German; policy description translated here. No crawler-specific IP file linked by the operator.",
      "checked": "2026-11-10"
    },
    {
      "operator": "Meta",
      "token": "meta-externalfetcher",
      "purpose": "User-triggered fetch",
      "docKey": "meta",
      "doc": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "policy": "May bypass robots.txt; user requests",
      "ipKey": "",
      "ipUrl": "",
      "kind": "agent",
      "robots": "meta-externalfetcher",
      "note": "Operator wording: \u201ckann die robots.txt-Datei umgehen\u201d. No crawler-specific IP file linked; broad Meta network ownership is not proof of this bot.",
      "checked": "2026-11-10"
    },
    {
      "operator": "Amazon",
      "token": "Amazonbot",
      "purpose": "Products and services; may train AI",
      "docKey": "amazon",
      "doc": "https://developer.amazon.com/amazonbot",
      "policy": "Yes; \u201crespect the Robots Exclusion Protocol\u201d",
      "ipKey": "ip_amazon",
      "ipUrl": "https://developer.amazon.com/amazonbot/ip-addresses/",
      "kind": "agent",
      "robots": "Amazonbot",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "Amazon",
      "token": "Amzn-SearchBot",
      "purpose": "Search index",
      "docKey": "amazon",
      "doc": "https://developer.amazon.com/amazonbot",
      "policy": "Yes; separate search control",
      "ipKey": "ip_amazon_search",
      "ipUrl": "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/",
      "kind": "agent",
      "robots": "Amzn-SearchBot",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "Amazon",
      "token": "Amzn-User",
      "purpose": "User-triggered fetch",
      "docKey": "amazon",
      "doc": "https://developer.amazon.com/amazonbot",
      "policy": "\u201cmay not follow all robots.txt directives\u201d",
      "ipKey": "ip_amazon_user",
      "ipUrl": "https://developer.amazon.com/amazonbot/live-ip-addresses/",
      "kind": "agent",
      "robots": "Amzn-User",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "ByteDance",
      "token": "Bytespider",
      "purpose": "AI crawler; precise purpose not stated",
      "docKey": "byteplus",
      "doc": "https://docs.byteplus.com/en/docs/byteplus-cdn/ai-crawler-traffic-monitoring_en-us",
      "policy": "No operator compliance statement found",
      "ipKey": "",
      "ipUrl": "",
      "kind": "reported",
      "robots": "Bytespider",
      "note": "BytePlus lists the name as an AI crawler. No operator training-purpose claim or dedicated IP verification list found; detection is a claimed token only.",
      "checked": "2026-11-10"
    },
    {
      "operator": "Common Crawl",
      "token": "CCBot",
      "purpose": "Open web corpus; downstream research and training",
      "docKey": "commoncrawl",
      "doc": "https://www.commoncrawl.org/ccbot",
      "policy": "Yes; robots exclusion example",
      "ipKey": "ip_commoncrawl",
      "ipUrl": "https://index.commoncrawl.org/ccbot.json",
      "kind": "agent",
      "robots": "CCBot",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "DuckDuckGo",
      "token": "DuckAssistBot",
      "purpose": "Real-time AI answer retrieval; no training",
      "docKey": "duck",
      "doc": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
      "policy": "Yes; \u201cafter 72 hours\u201d",
      "ipKey": "ip_duck",
      "ipUrl": "https://duckduckgo.com/duckassistbot.json",
      "kind": "agent",
      "robots": "DuckAssistBot",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "Mistral",
      "token": "MistralAI-User",
      "purpose": "User-triggered fetch",
      "docKey": "mistral",
      "doc": "https://docs.mistral.ai/robots",
      "policy": "No explicit compliance statement",
      "ipKey": "ip_mistral_user",
      "ipUrl": "https://mistral.ai/mistralai-user-ips.json",
      "kind": "agent",
      "robots": "MistralAI-User",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "Mistral",
      "token": "MistralAI-Index",
      "purpose": "Search index",
      "docKey": "mistral",
      "doc": "https://docs.mistral.ai/robots",
      "policy": "No explicit compliance statement",
      "ipKey": "ip_mistral_index",
      "ipUrl": "https://mistral.ai/mistralai-index-ips.json",
      "kind": "agent",
      "robots": "MistralAI-Index",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "Mistral",
      "token": "MistralAI-Training",
      "purpose": "Training",
      "docKey": "mistral",
      "doc": "https://docs.mistral.ai/robots",
      "policy": "\u201cWebmasters can disallow this user agent\u201d",
      "ipKey": "",
      "ipUrl": "",
      "kind": "agent",
      "robots": "MistralAI-Training",
      "note": "",
      "checked": "2026-11-10"
    },
    {
      "operator": "Unconfirmed (claimed Cohere)",
      "token": "cohere-ai",
      "purpose": "Not operator-documented",
      "docKey": "",
      "doc": "",
      "policy": "No operator statement verified",
      "ipKey": "",
      "ipUrl": "",
      "kind": "unconfirmed",
      "robots": "cohere-ai",
      "note": "Name requested for comparison; excluded from documented-agent counts and default policy recipes. No official current token documentation found.",
      "checked": "2026-11-10"
    },
    {
      "operator": "Unconfirmed (claimed Cohere)",
      "token": "cohere-training-data-crawler",
      "purpose": "Not operator-documented",
      "docKey": "",
      "doc": "",
      "policy": "No operator statement verified",
      "ipKey": "",
      "ipUrl": "",
      "kind": "unconfirmed",
      "robots": "cohere-training-data-crawler",
      "note": "Unconfirmed historical name; not included in the policy controls.",
      "checked": "2026-11-10"
    }
  ]
}
