[ { "id": "claude-code", "name": "Claude Code", "operator": "Anthropic", "category": "agent", "kind": "coding", "runsOn": "user", "userAgents": [ "^Claude-User \\(claude-code/(?[\\d.]+)", "^claude-code/(?[\\d.]+)" ], "docs": "https://code.claude.com/docs/en/tools-reference", "examples": [ "Claude-User (claude-code/2.1.308; +https://support.anthropic.com/)", "claude-code/2.1.107 (cli)" ], "notes": "Fetches from the user's machine, so Anthropic's IP ranges do not apply. Builds before roughly March 2026 sent a bare axios user agent.", "lastVerified": "2026-10-23" }, { "id": "github-copilot", "name": "GitHub Copilot", "operator": "GitHub", "category": "agent", "kind": "coding", "runsOn": "user", "userAgents": [ "\\bGitHubCopilot[\\w-]*" ], "examples": [ "GitHubCopilotRuntime-WebFetch" ], "lastVerified": "2026-10-23" }, { "id": "opencode", "name": "OpenCode", "operator": "SST", "category": "agent", "kind": "coding", "runsOn": "user", "userAgents": [ "\\bOpenCode-User/", "^opencode[/-]" ], "accept": [ "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1" ], "docs": "https://opencode.ai", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OpenCode-User/1.0; +https://opencode.ai", "opencode-smartfetch/1.0" ], "notes": "Some versions send a stock Chrome user agent; the Accept string still identifies them.", "lastVerified": "2026-10-23" }, { "id": "gemini-cli", "name": "Gemini CLI", "operator": "Google", "category": "agent", "kind": "coding", "runsOn": "operator", "userAgents": [ "\\bGoogleAgent-URLContext\\b" ], "examples": [ "GoogleAgent-URLContext" ], "notes": "URL context fetches run through Google infrastructure.", "lastVerified": "2026-10-23" }, { "id": "antigravity", "name": "Antigravity", "operator": "Google", "category": "agent", "kind": "coding", "runsOn": "operator", "userAgents": [ "\\bGoogle-Jetski-Antigravity\\b" ], "examples": [ "Google-Jetski-Antigravity" ], "lastVerified": "2026-10-23" }, { "id": "zcode", "name": "ZCode", "operator": "ZCode", "category": "agent", "kind": "coding", "runsOn": "user", "userAgents": [ "\\bZCode-WebFetch/" ], "docs": "https://zcode.ai", "examples": [ "ZCode-WebFetch/0.1 (+https://zcode.ai; coding-agent-cli)" ], "lastVerified": "2026-10-23" }, { "id": "deepseek-harness", "name": "DeepSeek harness", "operator": "DeepSeek", "category": "agent", "kind": "coding", "runsOn": "user", "userAgents": [ "^deepseek-harness/" ], "examples": [ "deepseek-harness/0.0.1 (+https://github.com/deepseek-ai)" ], "notes": "Observed in traffic; the operator has not documented what product sends it.", "lastVerified": "2026-10-23" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "operator": "OpenAI", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "\\bChatGPT-User/" ], "verify": { "ipRanges": [ "https://openai.com/chatgpt-user.json" ] }, "docs": "https://platform.openai.com/docs/bots", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot" ], "notes": "Codex has been reported to fetch with this user agent too, so it may not separate ChatGPT from Codex.", "lastVerified": "2026-10-23" }, { "id": "chatgpt-agent", "name": "ChatGPT agent", "operator": "OpenAI", "category": "agent", "kind": "browser", "runsOn": "operator", "userAgents": [], "signatureAgent": [ "chatgpt.com" ], "verify": { "webBotAuth": "https://chatgpt.com/.well-known/http-message-signatures-directory" }, "docs": "https://platform.openai.com/docs/bots", "examples": [], "notes": "Uses a regular browser user agent; identified by its Signature-Agent header.", "lastVerified": "2026-10-23" }, { "id": "claude-user", "name": "Claude-User", "operator": "Anthropic", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "compatible; Claude-User/", "^Claude-User$", "\\bClaude-Web\\b" ], "verify": { "ipRanges": [ "https://claude.com/crawling/bots.json" ] }, "docs": "https://support.claude.com/en/articles/8896518", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +claude-user@anthropic.com)", "Claude-User", "Claude-Web/1.0 (web crawler; +https://www.anthropic.com/; bots@anthropic.com)" ], "notes": "Fetches made by Claude apps on Anthropic infrastructure. Claude Code uses a different, more specific user agent. The bare `Claude-User` and older `Claude-Web` user agents are included; verify them by IP.", "lastVerified": "2026-10-23" }, { "id": "perplexity-user", "name": "Perplexity-User", "operator": "Perplexity", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "\\bPerplexity-User/", "\\bPerplexityUser/" ], "verify": { "ipRanges": [ "https://www.perplexity.ai/perplexity-user.json" ] }, "docs": "https://docs.perplexity.ai/guides/bots", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityUser/1.0; +https://perplexity.ai)" ], "lastVerified": "2026-10-23" }, { "id": "mistralai-user", "name": "MistralAI-User", "operator": "Mistral", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "\\bMistralAI-User/" ], "docs": "https://docs.mistral.ai/robots", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)" ], "lastVerified": "2026-10-23" }, { "id": "duckassistbot", "name": "DuckAssistBot", "operator": "DuckDuckGo", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "\\bDuckAssistBot/" ], "verify": { "ipRanges": [ "https://duckduckgo.com/duckassistbot.json" ], "webBotAuth": "https://assistbot.duckduckgo.com/.well-known/http-message-signatures-directory" }, "docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "examples": [ "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)" ], "lastVerified": "2026-10-23" }, { "id": "google-notebooklm", "name": "Google NotebookLM", "operator": "Google", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "\\bGoogle-NotebookLM\\b" ], "verify": { "ipRanges": [ "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json" ] }, "examples": [ "Google-NotebookLM", "Mozilla/5.0 (compatible; Google-NotebookLM; +https://notebooklm.google.com/)" ], "lastVerified": "2026-10-23" }, { "id": "google-agent", "name": "Google-Agent", "operator": "Google", "category": "agent", "kind": "browser", "runsOn": "operator", "userAgents": [ "\\bGoogle-Agent\\b" ], "signatureAgent": [ "agent.bot.goog" ], "verify": { "ipRanges": [ "https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json" ], "webBotAuth": "https://agent.bot.goog/.well-known/http-message-signatures-directory" }, "examples": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent) Chrome/155.0.8059.79 Safari/537.36" ], "notes": "Only some requests are signed; IP ranges are the primary check.", "lastVerified": "2026-10-23", "docs": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent" }, { "id": "exa-user", "name": "ExaUser", "operator": "Exa", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "\\bExaUser/" ], "signatureAgent": [ "crawler.exa.ai" ], "verify": { "webBotAuth": "https://crawler.exa.ai/.well-known/http-message-signatures-directory" }, "docs": "https://crawler.exa.ai/exauser", "examples": [ "Mozilla/5.0 (compatible; ExaUser/1.0; +https://crawler.exa.ai/exauser)" ], "notes": "Exa fetches on demand for other AI products, reportedly including Cursor. The product behind a given request is not identified.", "lastVerified": "2026-10-23" }, { "id": "grok-agent", "name": "Grok agent", "operator": "xAI", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "\\bgrok-agent/" ], "examples": [ "Mozilla/5.0 (compatible; grok-agent/1.0; +https://x.ai)" ], "lastVerified": "2026-10-23" }, { "id": "meta-externalfetcher", "name": "Meta-ExternalFetcher", "operator": "Meta", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "\\bmeta-externalfetcher/" ], "docs": "https://developers.facebook.com/docs/sharing/webmasters/crawler", "examples": [ "Mozilla/5.0 (compatible; Meta-ExternalFetcher/1.0; +https://developers.facebook.com/docs/sharing/webmasters/crawler)" ], "lastVerified": "2026-10-23" }, { "id": "amazon-quick", "name": "Amazon Quick", "operator": "Amazon", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "^amazon-Quick-on-behalf-of-(?[\\w-]+)" ], "examples": [ "amazon-Quick-on-behalf-of-0a1b2c3d" ], "notes": "Renders JavaScript, so it also fires client-side analytics.", "lastVerified": "2026-10-23" }, { "id": "amazon-agentcore-browser", "name": "Amazon Bedrock AgentCore Browser", "operator": "Amazon", "category": "agent", "kind": "browser", "runsOn": "operator", "userAgents": [ "\\bAmazon-Bedrock-AgentCore-Browser\\b" ], "examples": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Amazon-Bedrock-AgentCore-Browser/1.0 (Chromium; +https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html)" ], "lastVerified": "2026-10-23", "docs": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html" }, { "id": "minimax-agent", "name": "MiniMax Agent", "operator": "MiniMax", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "^MiniMaxAgent\\b" ], "examples": [ "MiniMaxAgent" ], "lastVerified": "2026-10-23" }, { "id": "browserbase", "name": "Browserbase", "operator": "Browserbase", "category": "agent", "kind": "browser", "runsOn": "operator", "userAgents": [], "signatureAgent": [ "browserbase.com" ], "verify": { "webBotAuth": "https://browserbase.com/.well-known/http-message-signatures-directory" }, "examples": [], "notes": "Hosted browsers for other companies' agents. A signature proves Browserbase, not the agent running on it.", "lastVerified": "2026-10-23" }, { "id": "manus", "name": "Manus", "operator": "Manus", "category": "agent", "kind": "browser", "runsOn": "operator", "userAgents": [], "signatureAgent": [ "api.manus.im" ], "verify": { "webBotAuth": "https://api.manus.im/.well-known/http-message-signatures-directory" }, "examples": [], "lastVerified": "2026-10-23" }, { "id": "cursor", "name": "Cursor", "operator": "Anysphere", "category": "agent", "kind": "coding", "runsOn": "user", "userAgents": [ "^Cursor/(?[\\d.]+) \\(", "\\bCursor/(?[\\d.]+) Chrome/[\\d.]+ Electron/" ], "examples": [ "Cursor/3.0.16 (darwin arm64)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Cursor/3.0.16 Chrome/142.0.7444.265 Electron/39.8.1 Safari/537.36" ], "notes": "Requests made by the Cursor app itself. Its web fetch tool has been reported to go through Exa as ExaUser.", "lastVerified": "2026-10-23" }, { "id": "qoder", "name": "Qoder CLI", "operator": "Alibaba", "category": "agent", "kind": "coding", "runsOn": "user", "userAgents": [ "^qodercli(cn)?/(?[\\d.]+)" ], "docs": "https://qoder.com", "examples": [ "qodercli/1.1.69 (+https://qoder.com)", "qoderclicn/1.1.69 (+https://qoder.cn)" ], "lastVerified": "2026-10-23" }, { "id": "kiro", "name": "Kiro", "operator": "Amazon", "category": "agent", "kind": "coding", "runsOn": "operator", "userAgents": [ "^KiroServer/" ], "docs": "https://kiro.dev", "examples": [ "KiroServer/1.0" ], "notes": "Observed in traffic. The KiroServer token suggests Kiro fetches pages from its own servers rather than the user's machine; Amazon has not published IP ranges for it.", "lastVerified": "2026-11-09" }, { "id": "augment", "name": "Augment", "operator": "Augment Code", "category": "agent", "kind": "coding", "runsOn": "user", "userAgents": [ "^Augment-WebFetch/" ], "docs": "https://www.augmentcode.com", "examples": [ "Augment-WebFetch/1.0" ], "notes": "Observed in traffic from Augment's web fetch tool.", "lastVerified": "2026-11-09" }, { "id": "qwen-code", "name": "Qwen Code", "operator": "Alibaba", "category": "agent", "kind": "coding", "runsOn": "user", "userAgents": [ "^QwenCode/(?[\\d.]+)" ], "docs": "https://github.com/QwenLM/qwen-code", "examples": [ "QwenCode/0.27.0 (linux; x64)", "QwenCode/0.26.9 (darwin; arm64)", "QwenCode/0.26.9 (win32; x64)" ], "lastVerified": "2026-11-09" }, { "id": "shap-user", "name": "Shap-User", "operator": "Shap", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "\\bShap-User/" ], "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Shap-User/0.1.0" ], "lastVerified": "2026-10-23" }, { "id": "kimi-user", "name": "Kimi-User", "operator": "Moonshot AI", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "\\bKimi-User/" ], "docs": "https://www.kimi.com/policies/kimi-crawlers", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Kimi-User/1.0; +https://www.kimi.com/policies/kimi-crawlers" ], "lastVerified": "2026-10-23" }, { "id": "liner-user", "name": "Liner-User", "operator": "Liner", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "\\bLiner-User/" ], "docs": "https://liner-user.liner.com/docs/liner-user", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Liner-User/1.0; +https://liner-user.liner.com/docs/liner-user)" ], "lastVerified": "2026-10-23" }, { "id": "gemini-deep-research", "name": "Gemini Deep Research", "operator": "Google", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "\\bGemini-Deep-Research\\b" ], "docs": "https://gemini.google/overview/deep-research/", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Gemini-Deep-Research; +https://gemini.google/overview/deep-research/) Chrome/135.0.0.0 Safari/537.36" ], "lastVerified": "2026-10-23" }, { "id": "amzn-user", "name": "Amzn-User", "operator": "Amazon", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "\\bAmzn-User/" ], "docs": "https://developer.amazon.com/amazonbot", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-User/0.1) Chrome/W.X.Y.Z Safari/537.36" ], "lastVerified": "2026-10-23" }, { "id": "kagi-fetcher", "name": "kagi-fetcher", "operator": "Kagi", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "^kagi-fetcher\\b" ], "examples": [ "kagi-fetcher/1.0" ], "lastVerified": "2026-10-23" }, { "id": "tabstack", "name": "Tabstack", "operator": "Mozilla", "category": "agent", "kind": "assistant", "runsOn": "operator", "userAgents": [ "^Mozilla-Tabstack/" ], "docs": "https://tabstack.ai", "examples": [ "Mozilla-Tabstack/1.0 (+https://tabstack.ai)" ], "notes": "Web access API for agents; the product behind a request is not identified.", "lastVerified": "2026-10-23" }, { "id": "gptbot", "name": "GPTBot", "operator": "OpenAI", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "\\bGPTBot/" ], "verify": { "ipRanges": [ "https://openai.com/gptbot.json" ] }, "docs": "https://platform.openai.com/docs/bots", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.4; +https://openai.com/gptbot)" ], "lastVerified": "2026-10-23" }, { "id": "claudebot", "name": "ClaudeBot", "operator": "Anthropic", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "\\bClaudeBot/", "^claudebot$", "^anthropic-ai$" ], "verify": { "ipRanges": [ "https://claude.com/crawling/bots.json" ] }, "docs": "https://support.claude.com/en/articles/8896518", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)", "anthropic-ai" ], "lastVerified": "2026-10-23" }, { "id": "meta-externalagent", "name": "Meta-ExternalAgent", "operator": "Meta", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "\\bmeta-externalagent/" ], "docs": "https://developers.facebook.com/docs/sharing/webmasters/crawler", "examples": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 (compatible; meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler))" ], "lastVerified": "2026-10-23" }, { "id": "bytespider", "name": "Bytespider", "operator": "ByteDance", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "\\bBytespider\\b" ], "examples": [ "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)" ], "lastVerified": "2026-10-23" }, { "id": "ccbot", "name": "CCBot", "operator": "Common Crawl", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "\\bCCBot/" ], "verify": { "ipRanges": [ "https://index.commoncrawl.org/ccbot.json" ] }, "docs": "https://commoncrawl.org/ccbot", "examples": [ "CCBot/2.0 (https://commoncrawl.org/faq/)" ], "lastVerified": "2026-10-23" }, { "id": "cohere-ai", "name": "cohere-ai", "operator": "Cohere", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "\\bcohere-ai\\b", "\\bcohere-training-data-crawler\\b" ], "examples": [ "Mozilla/5.0 (compatible; cohere-ai; +https://cohere.com/crawler)" ], "lastVerified": "2026-10-23" }, { "id": "amazonbot", "name": "Amazonbot", "operator": "Amazon", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "\\bAmazonbot/" ], "docs": "https://developer.amazon.com/amazonbot", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot) Chrome/119.0.6045.214 Safari/537.36" ], "lastVerified": "2026-10-23" }, { "id": "googleother", "name": "GoogleOther", "operator": "Google", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "\\bGoogleOther\\b" ], "verify": { "ipRanges": [ "https://developers.google.com/static/crawling/ipranges/common-crawlers.json" ], "reverseDns": [ "googlebot.com", "google.com" ] }, "docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GoogleOther) Chrome/153.0.8010.52 Safari/537.36" ], "notes": "Google's general-purpose crawler for research and development, not search indexing.", "lastVerified": "2026-10-23" }, { "id": "elevenlabs-crawler", "name": "ElevenlabsCrawler", "operator": "ElevenLabs", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "\\bElevenlabsCrawler/" ], "docs": "https://elevenlabs.io/docs/bots", "examples": [ "Mozilla/5.0 (compatible; ElevenlabsCrawler/1.0; +https://elevenlabs.io/docs/bots)" ], "lastVerified": "2026-10-23" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "operator": "OpenAI", "category": "ai-crawler", "kind": "ai-search", "runsOn": "operator", "userAgents": [ "\\bOAI-SearchBot/" ], "verify": { "ipRanges": [ "https://openai.com/searchbot.json" ] }, "docs": "https://platform.openai.com/docs/bots", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot" ], "lastVerified": "2026-10-23" }, { "id": "claude-searchbot", "name": "Claude-SearchBot", "operator": "Anthropic", "category": "ai-crawler", "kind": "ai-search", "runsOn": "operator", "userAgents": [ "\\bClaude-SearchBot/" ], "verify": { "ipRanges": [ "https://claude.com/crawling/bots.json" ] }, "docs": "https://support.claude.com/en/articles/8896518", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-SearchBot/1.0; +searchbot@anthropic.com)" ], "lastVerified": "2026-10-23" }, { "id": "perplexitybot", "name": "PerplexityBot", "operator": "Perplexity", "category": "ai-crawler", "kind": "ai-search", "runsOn": "operator", "userAgents": [ "\\bPerplexityBot/" ], "verify": { "ipRanges": [ "https://www.perplexity.ai/perplexitybot.json" ] }, "docs": "https://docs.perplexity.ai/guides/bots", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)" ], "notes": "The published range file was last updated on 2025-02-07.", "lastVerified": "2026-10-23" }, { "id": "exa-searchbot", "name": "ExaSearchBot", "operator": "Exa", "category": "ai-crawler", "kind": "ai-search", "runsOn": "operator", "userAgents": [ "\\bExaSearchBot/" ], "signatureAgent": [ "crawler.exa.ai" ], "verify": { "webBotAuth": "https://crawler.exa.ai/.well-known/http-message-signatures-directory" }, "docs": "https://crawler.exa.ai/", "examples": [ "Mozilla/5.0 (compatible; ExaSearchBot/1.0; +https://crawler.exa.ai/)" ], "lastVerified": "2026-10-23" }, { "id": "shapbot", "name": "ShapBot", "operator": "Shap", "category": "ai-crawler", "kind": "ai-search", "runsOn": "operator", "userAgents": [ "\\bShapBot/" ], "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ShapBot/0.1.0" ], "lastVerified": "2026-10-23" }, { "id": "youbot", "name": "YouBot", "operator": "You.com", "category": "ai-crawler", "kind": "ai-search", "runsOn": "operator", "userAgents": [ "\\bYouBot\\b" ], "signatureAgent": [ "you.com" ], "verify": { "webBotAuth": "https://you.com/.well-known/http-message-signatures-directory" }, "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; YouBot/1.0; +https://docs.you.com/youbot; env:prod) Chrome/142.0.0.0 Safari/537.36" ], "lastVerified": "2026-10-23", "docs": "https://docs.you.com/youbot" }, { "id": "amazon-bedrock-kb", "name": "Amazon Bedrock Knowledge Bases", "operator": "Amazon", "category": "ai-crawler", "kind": "ingest", "runsOn": "operator", "userAgents": [ "^amazon-bedrock-knowledgebase-on-behalf-of-(?[\\w-]+)" ], "examples": [ "amazon-bedrock-knowledgebase-on-behalf-of-0a1b2c3d" ], "lastVerified": "2026-10-23" }, { "id": "amazon-kendra", "name": "Amazon Kendra", "operator": "Amazon", "category": "ai-crawler", "kind": "ingest", "runsOn": "operator", "userAgents": [ "^amazon-kendra\\b" ], "examples": [ "amazon-kendra" ], "lastVerified": "2026-10-23" }, { "id": "google-cloudvertexbot", "name": "Google-CloudVertexBot", "operator": "Google", "category": "ai-crawler", "kind": "ingest", "runsOn": "operator", "userAgents": [ "\\bGoogle-CloudVertexBot\\b" ], "verify": { "ipRanges": [ "https://developers.google.com/static/crawling/ipranges/special-crawlers.json" ] }, "examples": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.8010.52 Mobile Safari/537.36 (compatible; Google-CloudVertexBot; +https://cloud.google.com/enterprise-search)" ], "notes": "Crawls sites that their owners connect to Vertex AI agents.", "lastVerified": "2026-10-23" }, { "id": "kimibot", "name": "KimiBot", "operator": "Moonshot AI", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "\\bKimiBot/" ], "docs": "https://www.kimi.com/policies/kimi-crawlers", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; KimiBot/1.0; +https://www.kimi.com/policies/kimi-crawlers" ], "lastVerified": "2026-10-23" }, { "id": "mistralai-training", "name": "MistralAI-Training", "operator": "Mistral", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "\\bMistralAI-Training/" ], "docs": "https://docs.mistral.ai/robots", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Training/1.0; +https://docs.mistral.ai/robots)" ], "lastVerified": "2026-10-23" }, { "id": "facebookbot", "name": "FacebookBot", "operator": "Meta", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "\\bFacebookBot/" ], "docs": "https://developers.facebook.com/docs/sharing/webmasters/facebookbot/", "examples": [ "Mozilla/5.0 (compatible; FacebookBot/1.0; +https://developers.facebook.com/docs/sharing/webmasters/facebookbot/)" ], "lastVerified": "2026-10-23" }, { "id": "ai2bot", "name": "AI2Bot", "operator": "Allen Institute for AI", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "\\bAi2Bot(-Dolma)?\\b" ], "examples": [ "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)", "Mozilla/5.0 (compatible) Ai2Bot-Dolma (+https://www.allenai.org/crawler)" ], "lastVerified": "2026-10-23" }, { "id": "clueweb", "name": "ClueWeb-Crawler", "operator": "Carnegie Mellon University", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "^ClueWeb-Crawler/" ], "examples": [ "ClueWeb-Crawler/1.0 (+https://boston.lti.cs.cmu.edu/CMU-ClueWeb-Crawler/; mailto:cmu-clueweb-crawler@andrew.cmu.edu)" ], "notes": "Research web corpus widely used for language model research.", "lastVerified": "2026-10-23" }, { "id": "training-datasets", "name": "Dataset crawlers", "operator": "Various", "category": "ai-crawler", "kind": "training", "runsOn": "operator", "userAgents": [ "\\bTimpibot/", "^ICC-Crawler/", "\\bVelenPublicWebCrawler\\b", "\\bDiffbot/", "\\bimg2(dataset|downloader)\\b", "\\blaion-huggingface-processor\\b", "^webzio-extended\\b", "\\bImagesiftBot\\b", "\\bSBIntuitionsBot/" ], "examples": [ "Mozilla/5.0 (compatible; Timpibot/0.9; +http://www.timpi.io)", "ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0 (compatible; img2downloader; +https://github.com/rom1504/img2dataset)", "Mozilla/5.0 (compatible; laion-huggingface-processor; +https://laion.ai/)", "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)", "Mozilla/5.0 (compatible; SBIntuitionsBot/0.1; +https://www.sbintuitions.co.jp/bot/)" ], "notes": "Crawlers that collect web data for datasets and model training.", "lastVerified": "2026-10-23" }, { "id": "kimi-searchbot", "name": "Kimi-SearchBot", "operator": "Moonshot AI", "category": "ai-crawler", "kind": "ai-search", "runsOn": "operator", "userAgents": [ "\\bKimi-SearchBot/" ], "docs": "https://www.kimi.com/policies/kimi-crawlers", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Kimi-SearchBot/1.0; +https://www.kimi.com/policies/kimi-crawlers" ], "lastVerified": "2026-10-23" }, { "id": "mistralai-index", "name": "MistralAI-Index", "operator": "Mistral", "category": "ai-crawler", "kind": "ai-search", "runsOn": "operator", "userAgents": [ "\\bMistralAI-Index/" ], "docs": "https://docs.mistral.ai/robots", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Index/1.0; +https://docs.mistral.ai/robots)" ], "lastVerified": "2026-10-23" }, { "id": "linerbot", "name": "LinerBot", "operator": "Liner", "category": "ai-crawler", "kind": "ai-search", "runsOn": "operator", "userAgents": [ "\\bLinerBot/" ], "docs": "https://liner-bot.liner.com/docs/liner-bot", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; LinerBot/1.0; +https://liner-bot.liner.com/docs/liner-bot)" ], "lastVerified": "2026-10-23" }, { "id": "amzn-searchbot", "name": "Amzn-SearchBot", "operator": "Amazon", "category": "ai-crawler", "kind": "ai-search", "runsOn": "operator", "userAgents": [ "\\bAmzn-SearchBot/" ], "docs": "https://developer.amazon.com/amazonbot", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/0.1) Chrome/119.0.6045.214 Safari/537.36" ], "lastVerified": "2026-10-23" }, { "id": "phindbot", "name": "PhindBot", "operator": "Phind", "category": "ai-crawler", "kind": "ai-search", "runsOn": "operator", "userAgents": [ "\\bPhindBot\\b" ], "examples": [ "Mozilla/5.0 (compatible; PhindBot; +https://www.phind.com/)" ], "lastVerified": "2026-10-23" }, { "id": "linkupbot", "name": "LinkupBot", "operator": "Linkup", "category": "ai-crawler", "kind": "ai-search", "runsOn": "operator", "userAgents": [ "\\bLinkupBot/" ], "docs": "https://linkup.so/bot", "examples": [ "LinkupBot/1.0 (LinkupBot for web indexing; https://linkup.so/bot; bot@linkup.so)" ], "lastVerified": "2026-10-23" }, { "id": "tavilybot", "name": "TavilyBot", "operator": "Tavily", "category": "ai-crawler", "kind": "ai-search", "runsOn": "operator", "userAgents": [ "\\bTavilyBot\\b" ], "examples": [ "Mozilla/5.0 (compatible; TavilyBot; +https://tavily.com/)" ], "notes": "Search API used by agents.", "lastVerified": "2026-10-23" }, { "id": "context7", "name": "Context7-Crawler", "operator": "Upstash", "category": "ai-crawler", "kind": "ingest", "runsOn": "operator", "userAgents": [ "^Context7-Crawler/" ], "docs": "https://context7.com/crawler", "examples": [ "Context7-Crawler/1.0 (+https://context7.com/crawler)" ], "notes": "Indexes library documentation that coding agents query.", "lastVerified": "2026-10-23" }, { "id": "firecrawl", "name": "FirecrawlAgent", "operator": "Firecrawl", "category": "ai-crawler", "kind": "ingest", "runsOn": "operator", "userAgents": [ "\\bFirecrawlAgent\\b" ], "examples": [ "Mozilla/5.0 (compatible; FirecrawlAgent; +https://firecrawl.dev/)" ], "notes": "Scraping API used by AI pipelines and agents; the product behind a request is not identified.", "lastVerified": "2026-10-23" }, { "id": "cloudflare-ai-search", "name": "Cloudflare AI Search", "operator": "Cloudflare", "category": "ai-crawler", "kind": "ingest", "runsOn": "operator", "userAgents": [ "^Cloudflare-(AutoRAG|AI-Search)\\b" ], "docs": "https://developers.cloudflare.com/autorag", "examples": [ "Cloudflare-AutoRAG (https://developers.cloudflare.com/autorag; autorag@cloudflare.com)", "Cloudflare-AI-Search" ], "lastVerified": "2026-10-23" }, { "id": "azure-ai-searchbot", "name": "AzureAI-SearchBot", "operator": "Microsoft", "category": "ai-crawler", "kind": "ingest", "runsOn": "operator", "userAgents": [ "\\bAzureAI-SearchBot/" ], "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; AzureAI-SearchBot/1.0;" ], "lastVerified": "2026-10-23" }, { "id": "amazon-qbusiness", "name": "Amazon Q Business", "operator": "Amazon", "category": "ai-crawler", "kind": "ingest", "runsOn": "operator", "userAgents": [ "^amazon-QBusiness\\b" ], "examples": [ "amazon-QBusiness" ], "lastVerified": "2026-10-23" }, { "id": "atlassian-bot", "name": "atlassian-bot", "operator": "Atlassian", "category": "ai-crawler", "kind": "ingest", "runsOn": "operator", "userAgents": [ "\\batlassian-bot\\b" ], "examples": [ "Mozilla/5.0 (compatible; atlassian-bot; +https://www.atlassian.com/)" ], "lastVerified": "2026-10-23" }, { "id": "googlebot", "name": "Googlebot", "operator": "Google", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bGooglebot(-Image|-Video|-News|-Mobile)?/" ], "verify": { "ipRanges": [ "https://developers.google.com/static/crawling/ipranges/common-crawlers.json" ], "reverseDns": [ "googlebot.com", "google.com" ] }, "docs": "https://developers.google.com/search/docs/crawling-indexing/googlebot", "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/153.0.8010.36 Safari/537.36", "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)" ], "notes": "Also feeds AI Overviews and Gemini grounding. Google-Extended is a robots.txt token only and never appears in a user agent.", "lastVerified": "2026-10-23" }, { "id": "bingbot", "name": "Bingbot", "operator": "Microsoft", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bbingbot/" ], "verify": { "ipRanges": [ "https://www.bing.com/toolbox/bingbot.json" ], "reverseDns": [ "search.msn.com" ] }, "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/136.0.0.0 Safari/537.36" ], "notes": "Also feeds Copilot answers.", "lastVerified": "2026-10-23" }, { "id": "applebot", "name": "Applebot", "operator": "Apple", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bApplebot/" ], "verify": { "ipRanges": [ "https://search.developer.apple.com/applebot.json" ], "reverseDns": [ "applebot.apple.com" ] }, "docs": "https://support.apple.com/en-us/119829", "examples": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)" ], "notes": "Applebot-Extended is a robots.txt token only.", "lastVerified": "2026-10-23" }, { "id": "duckduckbot", "name": "DuckDuckBot", "operator": "DuckDuckGo", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bDuckDuckBot\\b" ], "examples": [ "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)" ], "lastVerified": "2026-10-23" }, { "id": "yandexbot", "name": "YandexBot", "operator": "Yandex", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bYandex(Bot|MobileBot|Images|News|Webmaster|AccessibilityBot|RenderResourcesBot)/" ], "verify": { "reverseDns": [ "yandex.ru", "yandex.net", "yandex.com" ] }, "examples": [ "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)" ], "lastVerified": "2026-10-23" }, { "id": "baiduspider", "name": "Baiduspider", "operator": "Baidu", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bBaiduspider(-render)?/" ], "verify": { "reverseDns": [ "baidu.com", "baidu.jp" ] }, "examples": [ "Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)" ], "lastVerified": "2026-10-23" }, { "id": "yisouspider", "name": "YisouSpider", "operator": "Alibaba (Shenma)", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bYisouSpider\\b" ], "examples": [ "YisouSpider", "Mozilla/5.0 (iPhone; CPU iPhone OS 16_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Mobile/15E148 YisouSpider/5.0 Safari/604.1" ], "lastVerified": "2026-10-23" }, { "id": "sogou", "name": "Sogou web spider", "operator": "Sogou", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bSogou web spider/" ], "examples": [ "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" ], "lastVerified": "2026-10-23" }, { "id": "petalbot", "name": "PetalBot", "operator": "Huawei", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bPetalBot\\b" ], "examples": [ "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (HTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)" ], "lastVerified": "2026-10-23" }, { "id": "meta-webindexer", "name": "Meta-WebIndexer", "operator": "Meta", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bmeta-webindexer/" ], "docs": "https://developers.facebook.com/docs/sharing/webmasters/crawler", "examples": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 (compatible; meta-webindexer/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler))" ], "lastVerified": "2026-10-23" }, { "id": "seznambot", "name": "SeznamBot", "operator": "Seznam", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bSeznamBot/" ], "examples": [ "Mozilla/5.0 (compatible; SeznamBot/4.0; +https://o-seznam.cz/napoveda/vyhledavani/en/seznambot-crawler/)" ], "lastVerified": "2026-10-23" }, { "id": "coveobot", "name": "Coveobot", "operator": "Coveo", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bCoveobot/" ], "examples": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) (compatible; Coveobot/2.0;+http://www.coveo.com/bot.html)" ], "lastVerified": "2026-10-23" }, { "id": "google-inspectiontool", "name": "Google-InspectionTool", "operator": "Google", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bGoogle-InspectionTool/" ], "verify": { "ipRanges": [ "https://developers.google.com/static/crawling/ipranges/common-crawlers.json" ], "reverseDns": [ "googlebot.com", "google.com" ] }, "examples": [ "Mozilla/5.0 (compatible; Google-InspectionTool/1.0)" ], "notes": "Search Console URL inspection and Rich Results tests.", "lastVerified": "2026-10-23" }, { "id": "tiktokspider", "name": "TikTokSpider", "operator": "ByteDance", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bTikTokSpider\\b" ], "examples": [ "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; TikTokSpider; ttspider-feedback@tiktok.com)" ], "lastVerified": "2026-10-23" }, { "id": "coccoc", "name": "coccocbot", "operator": "Cốc Cốc", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\b(image\\.?)?coccoc/" ], "examples": [ "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)", "coccoc/1.0 (http://help.coccoc.com/)" ], "lastVerified": "2026-10-23" }, { "id": "ibou", "name": "ibou-search", "operator": "ibou", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bibou-search/" ], "docs": "https://ibou.com/crawlers.html", "examples": [ "Mozilla/5.0 (compatible; ibou-search/2.0; +https://ibou.com/crawlers.html)" ], "lastVerified": "2026-10-23" }, { "id": "algolia-crawler", "name": "Algolia Crawler", "operator": "Algolia", "category": "crawler", "kind": "search", "runsOn": "operator", "userAgents": [ "\\bAlgolia Crawler\\b" ], "examples": [ "Algolia Crawler/v0.0.0" ], "notes": "Site search indexing, including DocSearch.", "lastVerified": "2026-10-23" }, { "id": "ahrefsbot", "name": "AhrefsBot", "operator": "Ahrefs", "category": "bot", "kind": "seo", "runsOn": "operator", "userAgents": [ "\\bAhrefs(Bot|SiteAudit)/" ], "examples": [ "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)" ], "lastVerified": "2026-10-23" }, { "id": "semrushbot", "name": "SemrushBot", "operator": "Semrush", "category": "bot", "kind": "seo", "runsOn": "operator", "userAgents": [ "\\bSemrushBot\\b", "\\bSiteAuditBot/" ], "examples": [ "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)" ], "lastVerified": "2026-10-23" }, { "id": "seo-crawlers", "name": "Other SEO crawlers", "operator": "Various", "category": "bot", "kind": "seo", "runsOn": "operator", "userAgents": [ "\\bBarkrowler/", "\\bScreaming Frog SEO Spider/", "\\bDataForSeoBot/", "\\bSERankingBacklinksBot\\b", "\\bMJ12bot/", "\\bDotBot/", "\\bBLEXBot/", "\\bserpstatbot/", "\\bHubSpot Crawler\\b" ], "examples": [ "Mozilla/5.0 (compatible; Barkrowler/0.9; +https://babbar.tech/crawler)", "Screaming Frog SEO Spider/24.0" ], "lastVerified": "2026-10-23" }, { "id": "link-previews", "name": "Link preview fetchers", "operator": "Various", "category": "bot", "kind": "preview", "runsOn": "operator", "userAgents": [ "\\bfacebookexternalhit/", "\\bSlackbot-LinkExpanding\\b", "\\bTwitterbot/", "\\bDiscordbot/", "\\bLinkedInBot/", "\\bTelegramBot\\b", "\\bWhatsApp\\b", "^Facebot/", "\\bBingPreview/", "\\bSkypeUriPreview\\b", "\\bgooglemessages/", "\\bGoogleImageProxy\\b", "\\bGoogle Web Preview\\b" ], "examples": [ "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)", "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)", "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)", "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)", "Facebot/1.0", "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; BingPreview/1.0b) like Gecko", "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36" ], "lastVerified": "2026-10-23" }, { "id": "ad-verification", "name": "Ad verification bots", "operator": "Various", "category": "bot", "kind": "ads", "runsOn": "operator", "userAgents": [ "\\bAdsBot-Google(-Mobile)?\\b", "\\bmeta-externalads/", "\\bOAI-AdsBot/", "\\bGoogle-Adwords\\b", "(^|[(;,] ?)Mediapartners-Google\\b", "^Google-Display-Ads-Bot\\b", "\\bGoogle-Ads-Conversions\\b" ], "examples": [ "meta-externalads/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 (compatible; OAI-AdsBot/1.0; +https://openai.com/adsbot)", "Mediapartners-Google", "Google-Display-Ads-Bot", "Mozilla/5.0 AppleWebKit/537.36 Chrome/139.0.7258.127 Safari/537.36 Google-Ads-Conversions" ], "lastVerified": "2026-10-23" }, { "id": "monitors", "name": "Uptime and link monitors", "operator": "Various", "category": "bot", "kind": "monitor", "runsOn": "user", "userAgents": [ "^lychee/", "^Consul Health Check$", "\\bUptimeRobot/", "\\bPingdom\\b", "\\bStatusCake\\b", "\\bChecklyHQ\\b", "\\bDatadog/Synthetics\\b", "\\bGoogle-Site-Verification/", "\\bGoogle Page Speed Insights\\b", "\\bGoogle Search Console\\b", "\\bGoogle-Structured-Data-Testing-Tool\\b", "\\bCloudflareObservatory/", "\\bCatchpoint\\b" ], "examples": [ "lychee/0.24.2", "Consul Health Check", "Mozilla/5.0 (compatible; Google-Site-Verification/1.0)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Search Console) Chrome/41.0.2272.118 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 CloudflareObservatory/1.0", "Mozilla/5.0 (X11; Linux x86_64; Catchpoint) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36" ], "lastVerified": "2026-10-23" }, { "id": "security-scanners", "name": "Security scanners", "operator": "Various", "category": "bot", "kind": "scanner", "runsOn": "operator", "userAgents": [ "\\bDetectify\\b", "\\bCensysInspect/", "\\bNuclei\\b", "\\bzgrab/", "\\bAcunetix\\b" ], "examples": [ "Mozilla/5.0 (compatible; Detectify) +https://detectify.com/bot/", "Mozilla/5.0 (Linux; U; Android 4.0.3; en-us; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 Acunetix-Deepscan/31" ], "lastVerified": "2026-10-23" }, { "id": "archivers", "name": "Web archivers", "operator": "Various", "category": "bot", "kind": "archiver", "runsOn": "operator", "userAgents": [ "\\barchive\\.org_bot\\b", "\\bia_archiver\\b", "^Turnitin\\b" ], "examples": [ "Turnitin (https://bit.ly/2UvnfoQ)" ], "lastVerified": "2026-10-23" }, { "id": "feedfetcher-google", "name": "FeedFetcher-Google", "operator": "Google", "category": "bot", "kind": "feed", "runsOn": "operator", "userAgents": [ "^FeedFetcher-Google\\b" ], "examples": [ "FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)" ], "lastVerified": "2026-10-23" } ]