A Node.js package and GitHub Action for evaluating MCP (Model Context Protocol) tool implementations using LLM-based scoring. This helps ensure your MCP server's tools are working correctly and performing well.
Setup
Choose your client, copy the config, then fill in any required environment variables.
claude mcp add evals --env OPENAI_API_KEY='<OPENAI_API_KEY>' -- npx mcp-eval path/to/your/evals.ts path/to/your/server.ts~/.codex/config.toml
[mcp_servers.evals]
command = "npx"
args = ["mcp-eval", "path/to/your/evals.ts", "path/to/your/server.ts"]
env = { OPENAI_API_KEY = "<OPENAI_API_KEY>" }
# codex mcp add evals --env OPENAI_API_KEY='<OPENAI_API_KEY>' -- npx mcp-eval path/to/your/evals.ts path/to/your/server.ts~/.cursor/mcp.json
{
"mcpServers": {
"evals": {
"command": "npx",
"args": [
"mcp-eval",
"path/to/your/evals.ts",
"path/to/your/server.ts"
],
"env": {
"OPENAI_API_KEY": "<OPENAI_API_KEY>"
}
}
}
}.vscode/mcp.json
{
"servers": {
"evals": {
"type": "stdio",
"command": "npx",
"args": [
"mcp-eval",
"path/to/your/evals.ts",
"path/to/your/server.ts"
],
"env": {
"OPENAI_API_KEY": "<OPENAI_API_KEY>"
}
}
}
}~/.gemini/settings.json
{
"mcpServers": {
"evals": {
"command": "npx",
"args": [
"mcp-eval",
"path/to/your/evals.ts",
"path/to/your/server.ts"
],
"env": {
"OPENAI_API_KEY": "<OPENAI_API_KEY>"
}
}
}
}claude_desktop_config.json
{
"mcpServers": {
"evals": {
"command": "npx",
"args": [
"mcp-eval",
"path/to/your/evals.ts",
"path/to/your/server.ts"
],
"env": {
"OPENAI_API_KEY": "<OPENAI_API_KEY>"
}
}
}
}.trae/mcp.json
{
"mcpServers": {
"evals": {
"command": "npx",
"args": [
"mcp-eval",
"path/to/your/evals.ts",
"path/to/your/server.ts"
],
"env": {
"OPENAI_API_KEY": "<OPENAI_API_KEY>"
}
}
}
}Environment variables
OPENAI_API_KEY
Replace the placeholders with your own credentials and keep them out of version control.