1804 lines
50 KiB
JSON
1804 lines
50 KiB
JSON
[
|
|
{
|
|
"id": "claude-code",
|
|
"name": "Claude Code",
|
|
"operator": "Anthropic",
|
|
"category": "agent",
|
|
"kind": "coding",
|
|
"runsOn": "user",
|
|
"userAgents": [
|
|
"^Claude-User \\(claude-code/(?<version>[\\d.]+)",
|
|
"^claude-code/(?<version>[\\d.]+)"
|
|
],
|
|
"docs": "https://code.claude.com/docs/en/tools-reference",
|
|
"examples": [
|
|
"Claude-User (claude-code/2.1.308; +https://support.anthropic.com/)",
|
|
"claude-code/2.1.107 (cli)"
|
|
],
|
|
"notes": "Fetches from the user's machine, so Anthropic's IP ranges do not apply. Builds before roughly March 2026 sent a bare axios user agent.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "github-copilot",
|
|
"name": "GitHub Copilot",
|
|
"operator": "GitHub",
|
|
"category": "agent",
|
|
"kind": "coding",
|
|
"runsOn": "user",
|
|
"userAgents": [
|
|
"\\bGitHubCopilot[\\w-]*"
|
|
],
|
|
"examples": [
|
|
"GitHubCopilotRuntime-WebFetch"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "opencode",
|
|
"name": "OpenCode",
|
|
"operator": "SST",
|
|
"category": "agent",
|
|
"kind": "coding",
|
|
"runsOn": "user",
|
|
"userAgents": [
|
|
"\\bOpenCode-User/",
|
|
"^opencode[/-]"
|
|
],
|
|
"accept": [
|
|
"text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1"
|
|
],
|
|
"docs": "https://opencode.ai",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OpenCode-User/1.0; +https://opencode.ai",
|
|
"opencode-smartfetch/1.0"
|
|
],
|
|
"notes": "Some versions send a stock Chrome user agent; the Accept string still identifies them.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "gemini-cli",
|
|
"name": "Gemini CLI",
|
|
"operator": "Google",
|
|
"category": "agent",
|
|
"kind": "coding",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bGoogleAgent-URLContext\\b"
|
|
],
|
|
"examples": [
|
|
"GoogleAgent-URLContext"
|
|
],
|
|
"notes": "URL context fetches run through Google infrastructure.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "antigravity",
|
|
"name": "Antigravity",
|
|
"operator": "Google",
|
|
"category": "agent",
|
|
"kind": "coding",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bGoogle-Jetski-Antigravity\\b"
|
|
],
|
|
"examples": [
|
|
"Google-Jetski-Antigravity"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "zcode",
|
|
"name": "ZCode",
|
|
"operator": "ZCode",
|
|
"category": "agent",
|
|
"kind": "coding",
|
|
"runsOn": "user",
|
|
"userAgents": [
|
|
"\\bZCode-WebFetch/"
|
|
],
|
|
"docs": "https://zcode.ai",
|
|
"examples": [
|
|
"ZCode-WebFetch/0.1 (+https://zcode.ai; coding-agent-cli)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "deepseek-harness",
|
|
"name": "DeepSeek harness",
|
|
"operator": "DeepSeek",
|
|
"category": "agent",
|
|
"kind": "coding",
|
|
"runsOn": "user",
|
|
"userAgents": [
|
|
"^deepseek-harness/"
|
|
],
|
|
"examples": [
|
|
"deepseek-harness/0.0.1 (+https://github.com/deepseek-ai)"
|
|
],
|
|
"notes": "Observed in traffic; the operator has not documented what product sends it.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "chatgpt-user",
|
|
"name": "ChatGPT-User",
|
|
"operator": "OpenAI",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bChatGPT-User/"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://openai.com/chatgpt-user.json"
|
|
]
|
|
},
|
|
"docs": "https://platform.openai.com/docs/bots",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot"
|
|
],
|
|
"notes": "Codex has been reported to fetch with this user agent too, so it may not separate ChatGPT from Codex.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "chatgpt-agent",
|
|
"name": "ChatGPT agent",
|
|
"operator": "OpenAI",
|
|
"category": "agent",
|
|
"kind": "browser",
|
|
"runsOn": "operator",
|
|
"userAgents": [],
|
|
"signatureAgent": [
|
|
"chatgpt.com"
|
|
],
|
|
"verify": {
|
|
"webBotAuth": "https://chatgpt.com/.well-known/http-message-signatures-directory"
|
|
},
|
|
"docs": "https://platform.openai.com/docs/bots",
|
|
"examples": [],
|
|
"notes": "Uses a regular browser user agent; identified by its Signature-Agent header.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "claude-user",
|
|
"name": "Claude-User",
|
|
"operator": "Anthropic",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"compatible; Claude-User/",
|
|
"^Claude-User$",
|
|
"\\bClaude-Web\\b"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://claude.com/crawling/bots.json"
|
|
]
|
|
},
|
|
"docs": "https://support.claude.com/en/articles/8896518",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +claude-user@anthropic.com)",
|
|
"Claude-User",
|
|
"Claude-Web/1.0 (web crawler; +https://www.anthropic.com/; bots@anthropic.com)"
|
|
],
|
|
"notes": "Fetches made by Claude apps on Anthropic infrastructure. Claude Code uses a different, more specific user agent. The bare `Claude-User` and older `Claude-Web` user agents are included; verify them by IP.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "perplexity-user",
|
|
"name": "Perplexity-User",
|
|
"operator": "Perplexity",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bPerplexity-User/",
|
|
"\\bPerplexityUser/"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://www.perplexity.ai/perplexity-user.json"
|
|
]
|
|
},
|
|
"docs": "https://docs.perplexity.ai/guides/bots",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)",
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityUser/1.0; +https://perplexity.ai)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "mistralai-user",
|
|
"name": "MistralAI-User",
|
|
"operator": "Mistral",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bMistralAI-User/"
|
|
],
|
|
"docs": "https://docs.mistral.ai/robots",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "duckassistbot",
|
|
"name": "DuckAssistBot",
|
|
"operator": "DuckDuckGo",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bDuckAssistBot/"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://duckduckgo.com/duckassistbot.json"
|
|
],
|
|
"webBotAuth": "https://assistbot.duckduckgo.com/.well-known/http-message-signatures-directory"
|
|
},
|
|
"docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
|
|
"examples": [
|
|
"DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "google-notebooklm",
|
|
"name": "Google NotebookLM",
|
|
"operator": "Google",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bGoogle-NotebookLM\\b"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json"
|
|
]
|
|
},
|
|
"examples": [
|
|
"Google-NotebookLM",
|
|
"Mozilla/5.0 (compatible; Google-NotebookLM; +https://notebooklm.google.com/)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "google-agent",
|
|
"name": "Google-Agent",
|
|
"operator": "Google",
|
|
"category": "agent",
|
|
"kind": "browser",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bGoogle-Agent\\b"
|
|
],
|
|
"signatureAgent": [
|
|
"agent.bot.goog"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json"
|
|
],
|
|
"webBotAuth": "https://agent.bot.goog/.well-known/http-message-signatures-directory"
|
|
},
|
|
"examples": [
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent) Chrome/155.0.8059.79 Safari/537.36"
|
|
],
|
|
"notes": "Only some requests are signed; IP ranges are the primary check.",
|
|
"lastVerified": "2026-10-23",
|
|
"docs": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent"
|
|
},
|
|
{
|
|
"id": "exa-user",
|
|
"name": "ExaUser",
|
|
"operator": "Exa",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bExaUser/"
|
|
],
|
|
"signatureAgent": [
|
|
"crawler.exa.ai"
|
|
],
|
|
"verify": {
|
|
"webBotAuth": "https://crawler.exa.ai/.well-known/http-message-signatures-directory"
|
|
},
|
|
"docs": "https://crawler.exa.ai/exauser",
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; ExaUser/1.0; +https://crawler.exa.ai/exauser)"
|
|
],
|
|
"notes": "Exa fetches on demand for other AI products, reportedly including Cursor. The product behind a given request is not identified.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "grok-agent",
|
|
"name": "Grok agent",
|
|
"operator": "xAI",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bgrok-agent/"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; grok-agent/1.0; +https://x.ai)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "meta-externalfetcher",
|
|
"name": "Meta-ExternalFetcher",
|
|
"operator": "Meta",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bmeta-externalfetcher/"
|
|
],
|
|
"docs": "https://developers.facebook.com/docs/sharing/webmasters/crawler",
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; Meta-ExternalFetcher/1.0; +https://developers.facebook.com/docs/sharing/webmasters/crawler)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "amazon-quick",
|
|
"name": "Amazon Quick",
|
|
"operator": "Amazon",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"^amazon-Quick-on-behalf-of-(?<tenant>[\\w-]+)"
|
|
],
|
|
"examples": [
|
|
"amazon-Quick-on-behalf-of-0a1b2c3d"
|
|
],
|
|
"notes": "Renders JavaScript, so it also fires client-side analytics.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "amazon-agentcore-browser",
|
|
"name": "Amazon Bedrock AgentCore Browser",
|
|
"operator": "Amazon",
|
|
"category": "agent",
|
|
"kind": "browser",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bAmazon-Bedrock-AgentCore-Browser\\b"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Amazon-Bedrock-AgentCore-Browser/1.0 (Chromium; +https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html)"
|
|
],
|
|
"lastVerified": "2026-10-23",
|
|
"docs": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html"
|
|
},
|
|
{
|
|
"id": "minimax-agent",
|
|
"name": "MiniMax Agent",
|
|
"operator": "MiniMax",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"^MiniMaxAgent\\b"
|
|
],
|
|
"examples": [
|
|
"MiniMaxAgent"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "browserbase",
|
|
"name": "Browserbase",
|
|
"operator": "Browserbase",
|
|
"category": "agent",
|
|
"kind": "browser",
|
|
"runsOn": "operator",
|
|
"userAgents": [],
|
|
"signatureAgent": [
|
|
"browserbase.com"
|
|
],
|
|
"verify": {
|
|
"webBotAuth": "https://browserbase.com/.well-known/http-message-signatures-directory"
|
|
},
|
|
"examples": [],
|
|
"notes": "Hosted browsers for other companies' agents. A signature proves Browserbase, not the agent running on it.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "manus",
|
|
"name": "Manus",
|
|
"operator": "Manus",
|
|
"category": "agent",
|
|
"kind": "browser",
|
|
"runsOn": "operator",
|
|
"userAgents": [],
|
|
"signatureAgent": [
|
|
"api.manus.im"
|
|
],
|
|
"verify": {
|
|
"webBotAuth": "https://api.manus.im/.well-known/http-message-signatures-directory"
|
|
},
|
|
"examples": [],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "cursor",
|
|
"name": "Cursor",
|
|
"operator": "Anysphere",
|
|
"category": "agent",
|
|
"kind": "coding",
|
|
"runsOn": "user",
|
|
"userAgents": [
|
|
"^Cursor/(?<version>[\\d.]+) \\(",
|
|
"\\bCursor/(?<version>[\\d.]+) Chrome/[\\d.]+ Electron/"
|
|
],
|
|
"examples": [
|
|
"Cursor/3.0.16 (darwin arm64)",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Cursor/3.0.16 Chrome/142.0.7444.265 Electron/39.8.1 Safari/537.36"
|
|
],
|
|
"notes": "Requests made by the Cursor app itself. Its web fetch tool has been reported to go through Exa as ExaUser.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "qoder",
|
|
"name": "Qoder CLI",
|
|
"operator": "Alibaba",
|
|
"category": "agent",
|
|
"kind": "coding",
|
|
"runsOn": "user",
|
|
"userAgents": [
|
|
"^qodercli(cn)?/(?<version>[\\d.]+)"
|
|
],
|
|
"docs": "https://qoder.com",
|
|
"examples": [
|
|
"qodercli/1.1.69 (+https://qoder.com)",
|
|
"qoderclicn/1.1.69 (+https://qoder.cn)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "kiro",
|
|
"name": "Kiro",
|
|
"operator": "Amazon",
|
|
"category": "agent",
|
|
"kind": "coding",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"^KiroServer/"
|
|
],
|
|
"docs": "https://kiro.dev",
|
|
"examples": [
|
|
"KiroServer/1.0"
|
|
],
|
|
"notes": "Observed in traffic. The KiroServer token suggests Kiro fetches pages from its own servers rather than the user's machine; Amazon has not published IP ranges for it.",
|
|
"lastVerified": "2026-11-09"
|
|
},
|
|
{
|
|
"id": "augment",
|
|
"name": "Augment",
|
|
"operator": "Augment Code",
|
|
"category": "agent",
|
|
"kind": "coding",
|
|
"runsOn": "user",
|
|
"userAgents": [
|
|
"^Augment-WebFetch/"
|
|
],
|
|
"docs": "https://www.augmentcode.com",
|
|
"examples": [
|
|
"Augment-WebFetch/1.0"
|
|
],
|
|
"notes": "Observed in traffic from Augment's web fetch tool.",
|
|
"lastVerified": "2026-11-09"
|
|
},
|
|
{
|
|
"id": "qwen-code",
|
|
"name": "Qwen Code",
|
|
"operator": "Alibaba",
|
|
"category": "agent",
|
|
"kind": "coding",
|
|
"runsOn": "user",
|
|
"userAgents": [
|
|
"^QwenCode/(?<version>[\\d.]+)"
|
|
],
|
|
"docs": "https://github.com/QwenLM/qwen-code",
|
|
"examples": [
|
|
"QwenCode/0.27.0 (linux; x64)",
|
|
"QwenCode/0.26.9 (darwin; arm64)",
|
|
"QwenCode/0.26.9 (win32; x64)"
|
|
],
|
|
"lastVerified": "2026-11-09"
|
|
},
|
|
{
|
|
"id": "shap-user",
|
|
"name": "Shap-User",
|
|
"operator": "Shap",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bShap-User/"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Shap-User/0.1.0"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "kimi-user",
|
|
"name": "Kimi-User",
|
|
"operator": "Moonshot AI",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bKimi-User/"
|
|
],
|
|
"docs": "https://www.kimi.com/policies/kimi-crawlers",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Kimi-User/1.0; +https://www.kimi.com/policies/kimi-crawlers"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "liner-user",
|
|
"name": "Liner-User",
|
|
"operator": "Liner",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bLiner-User/"
|
|
],
|
|
"docs": "https://liner-user.liner.com/docs/liner-user",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Liner-User/1.0; +https://liner-user.liner.com/docs/liner-user)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "gemini-deep-research",
|
|
"name": "Gemini Deep Research",
|
|
"operator": "Google",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bGemini-Deep-Research\\b"
|
|
],
|
|
"docs": "https://gemini.google/overview/deep-research/",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Gemini-Deep-Research; +https://gemini.google/overview/deep-research/) Chrome/135.0.0.0 Safari/537.36"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "amzn-user",
|
|
"name": "Amzn-User",
|
|
"operator": "Amazon",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bAmzn-User/"
|
|
],
|
|
"docs": "https://developer.amazon.com/amazonbot",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-User/0.1) Chrome/W.X.Y.Z Safari/537.36"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "kagi-fetcher",
|
|
"name": "kagi-fetcher",
|
|
"operator": "Kagi",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"^kagi-fetcher\\b"
|
|
],
|
|
"examples": [
|
|
"kagi-fetcher/1.0"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "tabstack",
|
|
"name": "Tabstack",
|
|
"operator": "Mozilla",
|
|
"category": "agent",
|
|
"kind": "assistant",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"^Mozilla-Tabstack/"
|
|
],
|
|
"docs": "https://tabstack.ai",
|
|
"examples": [
|
|
"Mozilla-Tabstack/1.0 (+https://tabstack.ai)"
|
|
],
|
|
"notes": "Web access API for agents; the product behind a request is not identified.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "gptbot",
|
|
"name": "GPTBot",
|
|
"operator": "OpenAI",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bGPTBot/"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://openai.com/gptbot.json"
|
|
]
|
|
},
|
|
"docs": "https://platform.openai.com/docs/bots",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.4; +https://openai.com/gptbot)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "claudebot",
|
|
"name": "ClaudeBot",
|
|
"operator": "Anthropic",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bClaudeBot/",
|
|
"^claudebot$",
|
|
"^anthropic-ai$"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://claude.com/crawling/bots.json"
|
|
]
|
|
},
|
|
"docs": "https://support.claude.com/en/articles/8896518",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
|
|
"anthropic-ai"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "meta-externalagent",
|
|
"name": "Meta-ExternalAgent",
|
|
"operator": "Meta",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bmeta-externalagent/"
|
|
],
|
|
"docs": "https://developers.facebook.com/docs/sharing/webmasters/crawler",
|
|
"examples": [
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 (compatible; meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler))"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "bytespider",
|
|
"name": "Bytespider",
|
|
"operator": "ByteDance",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bBytespider\\b"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "ccbot",
|
|
"name": "CCBot",
|
|
"operator": "Common Crawl",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bCCBot/"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://index.commoncrawl.org/ccbot.json"
|
|
]
|
|
},
|
|
"docs": "https://commoncrawl.org/ccbot",
|
|
"examples": [
|
|
"CCBot/2.0 (https://commoncrawl.org/faq/)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "cohere-ai",
|
|
"name": "cohere-ai",
|
|
"operator": "Cohere",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bcohere-ai\\b",
|
|
"\\bcohere-training-data-crawler\\b"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; cohere-ai; +https://cohere.com/crawler)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "amazonbot",
|
|
"name": "Amazonbot",
|
|
"operator": "Amazon",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bAmazonbot/"
|
|
],
|
|
"docs": "https://developer.amazon.com/amazonbot",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot) Chrome/119.0.6045.214 Safari/537.36"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "googleother",
|
|
"name": "GoogleOther",
|
|
"operator": "Google",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bGoogleOther\\b"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://developers.google.com/static/crawling/ipranges/common-crawlers.json"
|
|
],
|
|
"reverseDns": [
|
|
"googlebot.com",
|
|
"google.com"
|
|
]
|
|
},
|
|
"docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GoogleOther) Chrome/153.0.8010.52 Safari/537.36"
|
|
],
|
|
"notes": "Google's general-purpose crawler for research and development, not search indexing.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "elevenlabs-crawler",
|
|
"name": "ElevenlabsCrawler",
|
|
"operator": "ElevenLabs",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bElevenlabsCrawler/"
|
|
],
|
|
"docs": "https://elevenlabs.io/docs/bots",
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; ElevenlabsCrawler/1.0; +https://elevenlabs.io/docs/bots)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "oai-searchbot",
|
|
"name": "OAI-SearchBot",
|
|
"operator": "OpenAI",
|
|
"category": "ai-crawler",
|
|
"kind": "ai-search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bOAI-SearchBot/"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://openai.com/searchbot.json"
|
|
]
|
|
},
|
|
"docs": "https://platform.openai.com/docs/bots",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "claude-searchbot",
|
|
"name": "Claude-SearchBot",
|
|
"operator": "Anthropic",
|
|
"category": "ai-crawler",
|
|
"kind": "ai-search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bClaude-SearchBot/"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://claude.com/crawling/bots.json"
|
|
]
|
|
},
|
|
"docs": "https://support.claude.com/en/articles/8896518",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-SearchBot/1.0; +searchbot@anthropic.com)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "perplexitybot",
|
|
"name": "PerplexityBot",
|
|
"operator": "Perplexity",
|
|
"category": "ai-crawler",
|
|
"kind": "ai-search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bPerplexityBot/"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://www.perplexity.ai/perplexitybot.json"
|
|
]
|
|
},
|
|
"docs": "https://docs.perplexity.ai/guides/bots",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)"
|
|
],
|
|
"notes": "The published range file was last updated on 2025-02-07.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "exa-searchbot",
|
|
"name": "ExaSearchBot",
|
|
"operator": "Exa",
|
|
"category": "ai-crawler",
|
|
"kind": "ai-search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bExaSearchBot/"
|
|
],
|
|
"signatureAgent": [
|
|
"crawler.exa.ai"
|
|
],
|
|
"verify": {
|
|
"webBotAuth": "https://crawler.exa.ai/.well-known/http-message-signatures-directory"
|
|
},
|
|
"docs": "https://crawler.exa.ai/",
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; ExaSearchBot/1.0; +https://crawler.exa.ai/)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "shapbot",
|
|
"name": "ShapBot",
|
|
"operator": "Shap",
|
|
"category": "ai-crawler",
|
|
"kind": "ai-search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bShapBot/"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ShapBot/0.1.0"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "youbot",
|
|
"name": "YouBot",
|
|
"operator": "You.com",
|
|
"category": "ai-crawler",
|
|
"kind": "ai-search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bYouBot\\b"
|
|
],
|
|
"signatureAgent": [
|
|
"you.com"
|
|
],
|
|
"verify": {
|
|
"webBotAuth": "https://you.com/.well-known/http-message-signatures-directory"
|
|
},
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; YouBot/1.0; +https://docs.you.com/youbot; env:prod) Chrome/142.0.0.0 Safari/537.36"
|
|
],
|
|
"lastVerified": "2026-10-23",
|
|
"docs": "https://docs.you.com/youbot"
|
|
},
|
|
{
|
|
"id": "amazon-bedrock-kb",
|
|
"name": "Amazon Bedrock Knowledge Bases",
|
|
"operator": "Amazon",
|
|
"category": "ai-crawler",
|
|
"kind": "ingest",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"^amazon-bedrock-knowledgebase-on-behalf-of-(?<tenant>[\\w-]+)"
|
|
],
|
|
"examples": [
|
|
"amazon-bedrock-knowledgebase-on-behalf-of-0a1b2c3d"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "amazon-kendra",
|
|
"name": "Amazon Kendra",
|
|
"operator": "Amazon",
|
|
"category": "ai-crawler",
|
|
"kind": "ingest",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"^amazon-kendra\\b"
|
|
],
|
|
"examples": [
|
|
"amazon-kendra"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "google-cloudvertexbot",
|
|
"name": "Google-CloudVertexBot",
|
|
"operator": "Google",
|
|
"category": "ai-crawler",
|
|
"kind": "ingest",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bGoogle-CloudVertexBot\\b"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://developers.google.com/static/crawling/ipranges/special-crawlers.json"
|
|
]
|
|
},
|
|
"examples": [
|
|
"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.8010.52 Mobile Safari/537.36 (compatible; Google-CloudVertexBot; +https://cloud.google.com/enterprise-search)"
|
|
],
|
|
"notes": "Crawls sites that their owners connect to Vertex AI agents.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "kimibot",
|
|
"name": "KimiBot",
|
|
"operator": "Moonshot AI",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bKimiBot/"
|
|
],
|
|
"docs": "https://www.kimi.com/policies/kimi-crawlers",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; KimiBot/1.0; +https://www.kimi.com/policies/kimi-crawlers"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "mistralai-training",
|
|
"name": "MistralAI-Training",
|
|
"operator": "Mistral",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bMistralAI-Training/"
|
|
],
|
|
"docs": "https://docs.mistral.ai/robots",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Training/1.0; +https://docs.mistral.ai/robots)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "facebookbot",
|
|
"name": "FacebookBot",
|
|
"operator": "Meta",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bFacebookBot/"
|
|
],
|
|
"docs": "https://developers.facebook.com/docs/sharing/webmasters/facebookbot/",
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; FacebookBot/1.0; +https://developers.facebook.com/docs/sharing/webmasters/facebookbot/)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "ai2bot",
|
|
"name": "AI2Bot",
|
|
"operator": "Allen Institute for AI",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bAi2Bot(-Dolma)?\\b"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)",
|
|
"Mozilla/5.0 (compatible) Ai2Bot-Dolma (+https://www.allenai.org/crawler)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "clueweb",
|
|
"name": "ClueWeb-Crawler",
|
|
"operator": "Carnegie Mellon University",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"^ClueWeb-Crawler/"
|
|
],
|
|
"examples": [
|
|
"ClueWeb-Crawler/1.0 (+https://boston.lti.cs.cmu.edu/CMU-ClueWeb-Crawler/; mailto:cmu-clueweb-crawler@andrew.cmu.edu)"
|
|
],
|
|
"notes": "Research web corpus widely used for language model research.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "training-datasets",
|
|
"name": "Dataset crawlers",
|
|
"operator": "Various",
|
|
"category": "ai-crawler",
|
|
"kind": "training",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bTimpibot/",
|
|
"^ICC-Crawler/",
|
|
"\\bVelenPublicWebCrawler\\b",
|
|
"\\bDiffbot/",
|
|
"\\bimg2(dataset|downloader)\\b",
|
|
"\\blaion-huggingface-processor\\b",
|
|
"^webzio-extended\\b",
|
|
"\\bImagesiftBot\\b",
|
|
"\\bSBIntuitionsBot/"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; Timpibot/0.9; +http://www.timpi.io)",
|
|
"ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)",
|
|
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)",
|
|
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0 (compatible; img2downloader; +https://github.com/rom1504/img2dataset)",
|
|
"Mozilla/5.0 (compatible; laion-huggingface-processor; +https://laion.ai/)",
|
|
"Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)",
|
|
"Mozilla/5.0 (compatible; SBIntuitionsBot/0.1; +https://www.sbintuitions.co.jp/bot/)"
|
|
],
|
|
"notes": "Crawlers that collect web data for datasets and model training.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "kimi-searchbot",
|
|
"name": "Kimi-SearchBot",
|
|
"operator": "Moonshot AI",
|
|
"category": "ai-crawler",
|
|
"kind": "ai-search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bKimi-SearchBot/"
|
|
],
|
|
"docs": "https://www.kimi.com/policies/kimi-crawlers",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Kimi-SearchBot/1.0; +https://www.kimi.com/policies/kimi-crawlers"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "mistralai-index",
|
|
"name": "MistralAI-Index",
|
|
"operator": "Mistral",
|
|
"category": "ai-crawler",
|
|
"kind": "ai-search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bMistralAI-Index/"
|
|
],
|
|
"docs": "https://docs.mistral.ai/robots",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Index/1.0; +https://docs.mistral.ai/robots)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "linerbot",
|
|
"name": "LinerBot",
|
|
"operator": "Liner",
|
|
"category": "ai-crawler",
|
|
"kind": "ai-search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bLinerBot/"
|
|
],
|
|
"docs": "https://liner-bot.liner.com/docs/liner-bot",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; LinerBot/1.0; +https://liner-bot.liner.com/docs/liner-bot)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "amzn-searchbot",
|
|
"name": "Amzn-SearchBot",
|
|
"operator": "Amazon",
|
|
"category": "ai-crawler",
|
|
"kind": "ai-search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bAmzn-SearchBot/"
|
|
],
|
|
"docs": "https://developer.amazon.com/amazonbot",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/0.1) Chrome/119.0.6045.214 Safari/537.36"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "phindbot",
|
|
"name": "PhindBot",
|
|
"operator": "Phind",
|
|
"category": "ai-crawler",
|
|
"kind": "ai-search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bPhindBot\\b"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; PhindBot; +https://www.phind.com/)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "linkupbot",
|
|
"name": "LinkupBot",
|
|
"operator": "Linkup",
|
|
"category": "ai-crawler",
|
|
"kind": "ai-search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bLinkupBot/"
|
|
],
|
|
"docs": "https://linkup.so/bot",
|
|
"examples": [
|
|
"LinkupBot/1.0 (LinkupBot for web indexing; https://linkup.so/bot; bot@linkup.so)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "tavilybot",
|
|
"name": "TavilyBot",
|
|
"operator": "Tavily",
|
|
"category": "ai-crawler",
|
|
"kind": "ai-search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bTavilyBot\\b"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; TavilyBot; +https://tavily.com/)"
|
|
],
|
|
"notes": "Search API used by agents.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "context7",
|
|
"name": "Context7-Crawler",
|
|
"operator": "Upstash",
|
|
"category": "ai-crawler",
|
|
"kind": "ingest",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"^Context7-Crawler/"
|
|
],
|
|
"docs": "https://context7.com/crawler",
|
|
"examples": [
|
|
"Context7-Crawler/1.0 (+https://context7.com/crawler)"
|
|
],
|
|
"notes": "Indexes library documentation that coding agents query.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "firecrawl",
|
|
"name": "FirecrawlAgent",
|
|
"operator": "Firecrawl",
|
|
"category": "ai-crawler",
|
|
"kind": "ingest",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bFirecrawlAgent\\b"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; FirecrawlAgent; +https://firecrawl.dev/)"
|
|
],
|
|
"notes": "Scraping API used by AI pipelines and agents; the product behind a request is not identified.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "cloudflare-ai-search",
|
|
"name": "Cloudflare AI Search",
|
|
"operator": "Cloudflare",
|
|
"category": "ai-crawler",
|
|
"kind": "ingest",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"^Cloudflare-(AutoRAG|AI-Search)\\b"
|
|
],
|
|
"docs": "https://developers.cloudflare.com/autorag",
|
|
"examples": [
|
|
"Cloudflare-AutoRAG (https://developers.cloudflare.com/autorag; autorag@cloudflare.com)",
|
|
"Cloudflare-AI-Search"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "azure-ai-searchbot",
|
|
"name": "AzureAI-SearchBot",
|
|
"operator": "Microsoft",
|
|
"category": "ai-crawler",
|
|
"kind": "ingest",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bAzureAI-SearchBot/"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; AzureAI-SearchBot/1.0;"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "amazon-qbusiness",
|
|
"name": "Amazon Q Business",
|
|
"operator": "Amazon",
|
|
"category": "ai-crawler",
|
|
"kind": "ingest",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"^amazon-QBusiness\\b"
|
|
],
|
|
"examples": [
|
|
"amazon-QBusiness"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "atlassian-bot",
|
|
"name": "atlassian-bot",
|
|
"operator": "Atlassian",
|
|
"category": "ai-crawler",
|
|
"kind": "ingest",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\batlassian-bot\\b"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; atlassian-bot; +https://www.atlassian.com/)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "googlebot",
|
|
"name": "Googlebot",
|
|
"operator": "Google",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bGooglebot(-Image|-Video|-News|-Mobile)?/"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://developers.google.com/static/crawling/ipranges/common-crawlers.json"
|
|
],
|
|
"reverseDns": [
|
|
"googlebot.com",
|
|
"google.com"
|
|
]
|
|
},
|
|
"docs": "https://developers.google.com/search/docs/crawling-indexing/googlebot",
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/153.0.8010.36 Safari/537.36",
|
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)"
|
|
],
|
|
"notes": "Also feeds AI Overviews and Gemini grounding. Google-Extended is a robots.txt token only and never appears in a user agent.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "bingbot",
|
|
"name": "Bingbot",
|
|
"operator": "Microsoft",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bbingbot/"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://www.bing.com/toolbox/bingbot.json"
|
|
],
|
|
"reverseDns": [
|
|
"search.msn.com"
|
|
]
|
|
},
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/136.0.0.0 Safari/537.36"
|
|
],
|
|
"notes": "Also feeds Copilot answers.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "applebot",
|
|
"name": "Applebot",
|
|
"operator": "Apple",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bApplebot/"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://search.developer.apple.com/applebot.json"
|
|
],
|
|
"reverseDns": [
|
|
"applebot.apple.com"
|
|
]
|
|
},
|
|
"docs": "https://support.apple.com/en-us/119829",
|
|
"examples": [
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)"
|
|
],
|
|
"notes": "Applebot-Extended is a robots.txt token only.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "duckduckbot",
|
|
"name": "DuckDuckBot",
|
|
"operator": "DuckDuckGo",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bDuckDuckBot\\b"
|
|
],
|
|
"examples": [
|
|
"DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "yandexbot",
|
|
"name": "YandexBot",
|
|
"operator": "Yandex",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bYandex(Bot|MobileBot|Images|News|Webmaster|AccessibilityBot|RenderResourcesBot)/"
|
|
],
|
|
"verify": {
|
|
"reverseDns": [
|
|
"yandex.ru",
|
|
"yandex.net",
|
|
"yandex.com"
|
|
]
|
|
},
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "baiduspider",
|
|
"name": "Baiduspider",
|
|
"operator": "Baidu",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bBaiduspider(-render)?/"
|
|
],
|
|
"verify": {
|
|
"reverseDns": [
|
|
"baidu.com",
|
|
"baidu.jp"
|
|
]
|
|
},
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "yisouspider",
|
|
"name": "YisouSpider",
|
|
"operator": "Alibaba (Shenma)",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bYisouSpider\\b"
|
|
],
|
|
"examples": [
|
|
"YisouSpider",
|
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 16_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Mobile/15E148 YisouSpider/5.0 Safari/604.1"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "sogou",
|
|
"name": "Sogou web spider",
|
|
"operator": "Sogou",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bSogou web spider/"
|
|
],
|
|
"examples": [
|
|
"Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "petalbot",
|
|
"name": "PetalBot",
|
|
"operator": "Huawei",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bPetalBot\\b"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (HTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "meta-webindexer",
|
|
"name": "Meta-WebIndexer",
|
|
"operator": "Meta",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bmeta-webindexer/"
|
|
],
|
|
"docs": "https://developers.facebook.com/docs/sharing/webmasters/crawler",
|
|
"examples": [
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 (compatible; meta-webindexer/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler))"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "seznambot",
|
|
"name": "SeznamBot",
|
|
"operator": "Seznam",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bSeznamBot/"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; SeznamBot/4.0; +https://o-seznam.cz/napoveda/vyhledavani/en/seznambot-crawler/)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "coveobot",
|
|
"name": "Coveobot",
|
|
"operator": "Coveo",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bCoveobot/"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) (compatible; Coveobot/2.0;+http://www.coveo.com/bot.html)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "google-inspectiontool",
|
|
"name": "Google-InspectionTool",
|
|
"operator": "Google",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bGoogle-InspectionTool/"
|
|
],
|
|
"verify": {
|
|
"ipRanges": [
|
|
"https://developers.google.com/static/crawling/ipranges/common-crawlers.json"
|
|
],
|
|
"reverseDns": [
|
|
"googlebot.com",
|
|
"google.com"
|
|
]
|
|
},
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; Google-InspectionTool/1.0)"
|
|
],
|
|
"notes": "Search Console URL inspection and Rich Results tests.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "tiktokspider",
|
|
"name": "TikTokSpider",
|
|
"operator": "ByteDance",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bTikTokSpider\\b"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; TikTokSpider; ttspider-feedback@tiktok.com)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "coccoc",
|
|
"name": "coccocbot",
|
|
"operator": "Cốc Cốc",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\b(image\\.?)?coccoc/"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)",
|
|
"coccoc/1.0 (http://help.coccoc.com/)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "ibou",
|
|
"name": "ibou-search",
|
|
"operator": "ibou",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bibou-search/"
|
|
],
|
|
"docs": "https://ibou.com/crawlers.html",
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; ibou-search/2.0; +https://ibou.com/crawlers.html)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "algolia-crawler",
|
|
"name": "Algolia Crawler",
|
|
"operator": "Algolia",
|
|
"category": "crawler",
|
|
"kind": "search",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bAlgolia Crawler\\b"
|
|
],
|
|
"examples": [
|
|
"Algolia Crawler/v0.0.0"
|
|
],
|
|
"notes": "Site search indexing, including DocSearch.",
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "ahrefsbot",
|
|
"name": "AhrefsBot",
|
|
"operator": "Ahrefs",
|
|
"category": "bot",
|
|
"kind": "seo",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bAhrefs(Bot|SiteAudit)/"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "semrushbot",
|
|
"name": "SemrushBot",
|
|
"operator": "Semrush",
|
|
"category": "bot",
|
|
"kind": "seo",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bSemrushBot\\b",
|
|
"\\bSiteAuditBot/"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "seo-crawlers",
|
|
"name": "Other SEO crawlers",
|
|
"operator": "Various",
|
|
"category": "bot",
|
|
"kind": "seo",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bBarkrowler/",
|
|
"\\bScreaming Frog SEO Spider/",
|
|
"\\bDataForSeoBot/",
|
|
"\\bSERankingBacklinksBot\\b",
|
|
"\\bMJ12bot/",
|
|
"\\bDotBot/",
|
|
"\\bBLEXBot/",
|
|
"\\bserpstatbot/",
|
|
"\\bHubSpot Crawler\\b"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; Barkrowler/0.9; +https://babbar.tech/crawler)",
|
|
"Screaming Frog SEO Spider/24.0"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "link-previews",
|
|
"name": "Link preview fetchers",
|
|
"operator": "Various",
|
|
"category": "bot",
|
|
"kind": "preview",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bfacebookexternalhit/",
|
|
"\\bSlackbot-LinkExpanding\\b",
|
|
"\\bTwitterbot/",
|
|
"\\bDiscordbot/",
|
|
"\\bLinkedInBot/",
|
|
"\\bTelegramBot\\b",
|
|
"\\bWhatsApp\\b",
|
|
"^Facebot/",
|
|
"\\bBingPreview/",
|
|
"\\bSkypeUriPreview\\b",
|
|
"\\bgooglemessages/",
|
|
"\\bGoogleImageProxy\\b",
|
|
"\\bGoogle Web Preview\\b"
|
|
],
|
|
"examples": [
|
|
"facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
|
|
"Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)",
|
|
"LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)",
|
|
"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)",
|
|
"Facebot/1.0",
|
|
"Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; BingPreview/1.0b) like Gecko",
|
|
"Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5",
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "ad-verification",
|
|
"name": "Ad verification bots",
|
|
"operator": "Various",
|
|
"category": "bot",
|
|
"kind": "ads",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bAdsBot-Google(-Mobile)?\\b",
|
|
"\\bmeta-externalads/",
|
|
"\\bOAI-AdsBot/",
|
|
"\\bGoogle-Adwords\\b",
|
|
"(^|[(;,] ?)Mediapartners-Google\\b",
|
|
"^Google-Display-Ads-Bot\\b",
|
|
"\\bGoogle-Ads-Conversions\\b"
|
|
],
|
|
"examples": [
|
|
"meta-externalads/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 (compatible; OAI-AdsBot/1.0; +https://openai.com/adsbot)",
|
|
"Mediapartners-Google",
|
|
"Google-Display-Ads-Bot",
|
|
"Mozilla/5.0 AppleWebKit/537.36 Chrome/139.0.7258.127 Safari/537.36 Google-Ads-Conversions"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "monitors",
|
|
"name": "Uptime and link monitors",
|
|
"operator": "Various",
|
|
"category": "bot",
|
|
"kind": "monitor",
|
|
"runsOn": "user",
|
|
"userAgents": [
|
|
"^lychee/",
|
|
"^Consul Health Check$",
|
|
"\\bUptimeRobot/",
|
|
"\\bPingdom\\b",
|
|
"\\bStatusCake\\b",
|
|
"\\bChecklyHQ\\b",
|
|
"\\bDatadog/Synthetics\\b",
|
|
"\\bGoogle-Site-Verification/",
|
|
"\\bGoogle Page Speed Insights\\b",
|
|
"\\bGoogle Search Console\\b",
|
|
"\\bGoogle-Structured-Data-Testing-Tool\\b",
|
|
"\\bCloudflareObservatory/",
|
|
"\\bCatchpoint\\b"
|
|
],
|
|
"examples": [
|
|
"lychee/0.24.2",
|
|
"Consul Health Check",
|
|
"Mozilla/5.0 (compatible; Google-Site-Verification/1.0)",
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Search Console) Chrome/41.0.2272.118 Safari/537.36",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 CloudflareObservatory/1.0",
|
|
"Mozilla/5.0 (X11; Linux x86_64; Catchpoint) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "security-scanners",
|
|
"name": "Security scanners",
|
|
"operator": "Various",
|
|
"category": "bot",
|
|
"kind": "scanner",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\bDetectify\\b",
|
|
"\\bCensysInspect/",
|
|
"\\bNuclei\\b",
|
|
"\\bzgrab/",
|
|
"\\bAcunetix\\b"
|
|
],
|
|
"examples": [
|
|
"Mozilla/5.0 (compatible; Detectify) +https://detectify.com/bot/",
|
|
"Mozilla/5.0 (Linux; U; Android 4.0.3; en-us; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 Acunetix-Deepscan/31"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "archivers",
|
|
"name": "Web archivers",
|
|
"operator": "Various",
|
|
"category": "bot",
|
|
"kind": "archiver",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"\\barchive\\.org_bot\\b",
|
|
"\\bia_archiver\\b",
|
|
"^Turnitin\\b"
|
|
],
|
|
"examples": [
|
|
"Turnitin (https://bit.ly/2UvnfoQ)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
},
|
|
{
|
|
"id": "feedfetcher-google",
|
|
"name": "FeedFetcher-Google",
|
|
"operator": "Google",
|
|
"category": "bot",
|
|
"kind": "feed",
|
|
"runsOn": "operator",
|
|
"userAgents": [
|
|
"^FeedFetcher-Google\\b"
|
|
],
|
|
"examples": [
|
|
"FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)"
|
|
],
|
|
"lastVerified": "2026-10-23"
|
|
}
|
|
]
|