Web crawler MCP server: crawl and scrape websites for SEO audits from Claude, ChatGPT, Cursor and Claude Code. Hosted remote MCP.
Setup
Choose your client, copy the config, then fill in any required environment variables.
claude mcp add --transport http web-crawler-by-insightfulpipe https://crawler.insightfulmcp.com/~/.codex/config.toml
[mcp_servers.web-crawler-by-insightfulpipe]
url = "https://crawler.insightfulmcp.com/"
# codex mcp add web-crawler-by-insightfulpipe --url https://crawler.insightfulmcp.com/~/.cursor/mcp.json
{
"mcpServers": {
"web-crawler-by-insightfulpipe": {
"url": "https://crawler.insightfulmcp.com/"
}
}
}.vscode/mcp.json
{
"servers": {
"web-crawler-by-insightfulpipe": {
"type": "http",
"url": "https://crawler.insightfulmcp.com/"
}
}
}~/.gemini/settings.json
{
"mcpServers": {
"web-crawler-by-insightfulpipe": {
"httpUrl": "https://crawler.insightfulmcp.com/"
}
}
}claude_desktop_config.json · Customize › Connectors › Add custom connector
{
"mcpServers": {
"web-crawler-by-insightfulpipe": {
"command": "npx",
"args": [
"-y",
"mcp-remote",
"https://crawler.insightfulmcp.com/"
]
}
}
}.trae/mcp.json
{
"mcpServers": {
"web-crawler-by-insightfulpipe": {
"url": "https://crawler.insightfulmcp.com/"
}
}
}