Bounded crawl-readiness check for agent access: robots.txt, sitemap.xml, llms.txt, homepage.
Setup
Choose your client, copy the config, then fill in any required environment variables.
claude mcp add --transport http agent-crawl-readiness https://agent-observatory-sensor.nolimit-observatory.workers.dev/s/agent-crawl-readiness/mcp~/.codex/config.toml
[mcp_servers.agent-crawl-readiness]
url = "https://agent-observatory-sensor.nolimit-observatory.workers.dev/s/agent-crawl-readiness/mcp"
# codex mcp add agent-crawl-readiness --url https://agent-observatory-sensor.nolimit-observatory.workers.dev/s/agent-crawl-readiness/mcp~/.cursor/mcp.json
{
"mcpServers": {
"agent-crawl-readiness": {
"url": "https://agent-observatory-sensor.nolimit-observatory.workers.dev/s/agent-crawl-readiness/mcp"
}
}
}.vscode/mcp.json
{
"servers": {
"agent-crawl-readiness": {
"type": "http",
"url": "https://agent-observatory-sensor.nolimit-observatory.workers.dev/s/agent-crawl-readiness/mcp"
}
}
}~/.gemini/settings.json
{
"mcpServers": {
"agent-crawl-readiness": {
"httpUrl": "https://agent-observatory-sensor.nolimit-observatory.workers.dev/s/agent-crawl-readiness/mcp"
}
}
}claude_desktop_config.json · Customize › Connectors › Add custom connector
{
"mcpServers": {
"agent-crawl-readiness": {
"command": "npx",
"args": [
"-y",
"mcp-remote",
"https://agent-observatory-sensor.nolimit-observatory.workers.dev/s/agent-crawl-readiness/mcp"
]
}
}
}.trae/mcp.json
{
"mcpServers": {
"agent-crawl-readiness": {
"url": "https://agent-observatory-sensor.nolimit-observatory.workers.dev/s/agent-crawl-readiness/mcp"
}
}
}