Open-source web scraper + LLM-powered structured extraction. PDF/DOCX, markdown, JSON-LD, microdata, commerce data, forms, 16 analytics-tracker detection. Structured errors with retryable flags. Adaptive Cheerio->Playwright. CLI, npm, REST API, and MCP server. AGPL-3.0.
Setup
Choose your client, copy the config, then fill in any required environment variables.
claude mcp add thecrawler -- npx -y thecrawler~/.codex/config.toml
[mcp_servers.thecrawler]
command = "npx"
args = ["-y", "thecrawler"]
# codex mcp add thecrawler -- npx -y thecrawler~/.cursor/mcp.json
{
"mcpServers": {
"thecrawler": {
"command": "npx",
"args": [
"-y",
"thecrawler"
]
}
}
}.vscode/mcp.json
{
"servers": {
"thecrawler": {
"type": "stdio",
"command": "npx",
"args": [
"-y",
"thecrawler"
]
}
}
}~/.gemini/settings.json
{
"mcpServers": {
"thecrawler": {
"command": "npx",
"args": [
"-y",
"thecrawler"
]
}
}
}claude_desktop_config.json
{
"mcpServers": {
"thecrawler": {
"command": "npx",
"args": [
"-y",
"thecrawler"
]
}
}
}.trae/mcp.json
{
"mcpServers": {
"thecrawler": {
"command": "npx",
"args": [
"-y",
"thecrawler"
]
}
}
}