Octocrawl: readable Markdown and checked fields from public web pages, for AI agents, RAG and research. Failures come back with a reason. Self-host with REST, SDK or MCP.
Setup
Choose your client, copy the config, then fill in any required environment variables.
Local
claude mcp add octocrawl --env W2L_API_TOKEN='<W2L_API_TOKEN>' -- npx -y @octocrawl/mcp~/.codex/config.toml
[mcp_servers.octocrawl]
command = "npx"
args = ["-y", "@octocrawl/mcp"]
env = { W2L_API_TOKEN = "<W2L_API_TOKEN>" }
# codex mcp add octocrawl --env W2L_API_TOKEN='<W2L_API_TOKEN>' -- npx -y @octocrawl/mcp~/.cursor/mcp.json
{
"mcpServers": {
"octocrawl": {
"command": "npx",
"args": [
"-y",
"@octocrawl/mcp"
],
"env": {
"W2L_API_TOKEN": "<W2L_API_TOKEN>"
}
}
}
}.vscode/mcp.json
{
"servers": {
"octocrawl": {
"type": "stdio",
"command": "npx",
"args": [
"-y",
"@octocrawl/mcp"
],
"env": {
"W2L_API_TOKEN": "<W2L_API_TOKEN>"
}
}
}
}~/.gemini/settings.json
{
"mcpServers": {
"octocrawl": {
"command": "npx",
"args": [
"-y",
"@octocrawl/mcp"
],
"env": {
"W2L_API_TOKEN": "<W2L_API_TOKEN>"
}
}
}
}claude_desktop_config.json
{
"mcpServers": {
"octocrawl": {
"command": "npx",
"args": [
"-y",
"@octocrawl/mcp"
],
"env": {
"W2L_API_TOKEN": "<W2L_API_TOKEN>"
}
}
}
}.trae/mcp.json
{
"mcpServers": {
"octocrawl": {
"command": "npx",
"args": [
"-y",
"@octocrawl/mcp"
],
"env": {
"W2L_API_TOKEN": "<W2L_API_TOKEN>"
}
}
}
}Remote
claude mcp add --transport http octocrawl https://mcp.octocrawl.dev/mcp~/.codex/config.toml
[mcp_servers.octocrawl]
url = "https://mcp.octocrawl.dev/mcp"
# codex mcp add octocrawl --url https://mcp.octocrawl.dev/mcp~/.cursor/mcp.json
{
"mcpServers": {
"octocrawl": {
"url": "https://mcp.octocrawl.dev/mcp"
}
}
}.vscode/mcp.json
{
"servers": {
"octocrawl": {
"type": "http",
"url": "https://mcp.octocrawl.dev/mcp"
}
}
}~/.gemini/settings.json
{
"mcpServers": {
"octocrawl": {
"httpUrl": "https://mcp.octocrawl.dev/mcp"
}
}
}claude_desktop_config.json · Customize › Connectors › Add custom connector
{
"mcpServers": {
"octocrawl": {
"command": "npx",
"args": [
"-y",
"mcp-remote",
"https://mcp.octocrawl.dev/mcp"
]
}
}
}.trae/mcp.json
{
"mcpServers": {
"octocrawl": {
"url": "https://mcp.octocrawl.dev/mcp"
}
}
}Environment variables
W2L_API_TOKEN
Replace the placeholders with your own credentials and keep them out of version control.