Ask before you fetch. Skip domains that will refuse your crawler, and never route around an HTTP 402 paywall. JS + Python, zero dependencies.
接入配置
选择客户端，复制配置，再按项目文档补齐环境变量。
claude mcp add --transport http crawl-census https://crawlcensus.com/mcp~/.codex/config.toml
[mcp_servers.crawl-census]
url = "https://crawlcensus.com/mcp"
# codex mcp add crawl-census --url https://crawlcensus.com/mcp~/.cursor/mcp.json
{
"mcpServers": {
"crawl-census": {
"url": "https://crawlcensus.com/mcp"
}
}
}.vscode/mcp.json
{
"servers": {
"crawl-census": {
"type": "http",
"url": "https://crawlcensus.com/mcp"
}
}
}~/.gemini/settings.json
{
"mcpServers": {
"crawl-census": {
"httpUrl": "https://crawlcensus.com/mcp"
}
}
}claude_desktop_config.json · Customize › Connectors › Add custom connector
{
"mcpServers": {
"crawl-census": {
"command": "npx",
"args": [
"-y",
"mcp-remote",
"https://crawlcensus.com/mcp"
]
}
}
}.trae/mcp.json
{
"mcpServers": {
"crawl-census": {
"url": "https://crawlcensus.com/mcp"
}
}
}