llmbenchmark-mcp 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmbenchmark_mcp-1.0.0/.cursor/mcp.json +11 -0
- llmbenchmark_mcp-1.0.0/.gitignore +4 -0
- llmbenchmark_mcp-1.0.0/Dockerfile +6 -0
- llmbenchmark_mcp-1.0.0/LICENSE +21 -0
- llmbenchmark_mcp-1.0.0/PKG-INFO +139 -0
- llmbenchmark_mcp-1.0.0/README.md +124 -0
- llmbenchmark_mcp-1.0.0/examples/prompts.md +12 -0
- llmbenchmark_mcp-1.0.0/glama.json +1 -0
- llmbenchmark_mcp-1.0.0/llmbenchmark_mcp.py +185 -0
- llmbenchmark_mcp-1.0.0/mcp.json +11 -0
- llmbenchmark_mcp-1.0.0/pyproject.toml +29 -0
- llmbenchmark_mcp-1.0.0/server.json +27 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 TAKE GROUP sp. z o.o.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: llmbenchmark-mcp
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Which LLM for which task: benchmark scores and API prices for 400+ models as an MCP server. Free, no API key.
|
|
5
|
+
Project-URL: Homepage, https://llmbenchmark.ai/api?utm_source=pypi&utm_medium=package&utm_campaign=mcp
|
|
6
|
+
Project-URL: Source, https://github.com/Take-Group/llmbenchmark-mcp
|
|
7
|
+
Author-email: "LLMBenchmark.ai" <hello@llmbenchmark.ai>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: chatgpt,claude,cursor,llm api pricing,llm benchmark,llm leaderboard,mcp server,model comparison
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Topic :: Software Development
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# LLMBenchmark.ai MCP server
|
|
17
|
+
|
|
18
|
+
Ask "which model should I use for this task?" and get an answer from benchmark scores and API prices of 400+ LLMs, updated daily.
|
|
19
|
+
|
|
20
|
+
- Free. No API key. 500 requests per IP per UTC day.
|
|
21
|
+
- Remote server, Streamable HTTP: `https://api.llmbenchmark.ai/mcp`
|
|
22
|
+
- Deterministic: no LLM call inside, the same task and the same data return the same answer.
|
|
23
|
+
- Every answer carries the data date, the source ("LLMBenchmark.ai") and links to the pages behind the numbers.
|
|
24
|
+
|
|
25
|
+
Docs: https://llmbenchmark.ai/api
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
**Claude Code**
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
claude mcp add --transport http llmbenchmark https://api.llmbenchmark.ai/mcp
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
**Cursor** (`.cursor/mcp.json`)
|
|
36
|
+
|
|
37
|
+
```json
|
|
38
|
+
{
|
|
39
|
+
"mcpServers": {
|
|
40
|
+
"llmbenchmark": { "url": "https://api.llmbenchmark.ai/mcp" }
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
**Codex** (`~/.codex/config.toml`)
|
|
46
|
+
|
|
47
|
+
```toml
|
|
48
|
+
[mcp_servers.llmbenchmark]
|
|
49
|
+
url = "https://api.llmbenchmark.ai/mcp"
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
**Claude Desktop** (`claude_desktop_config.json`, through the `mcp-remote` bridge)
|
|
53
|
+
|
|
54
|
+
```json
|
|
55
|
+
{
|
|
56
|
+
"mcpServers": {
|
|
57
|
+
"llmbenchmark": {
|
|
58
|
+
"command": "npx",
|
|
59
|
+
"args": ["-y", "mcp-remote", "https://api.llmbenchmark.ai/mcp"]
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
**Install as a local server (stdio)**
|
|
66
|
+
|
|
67
|
+
The package proxies to the hosted server, so clients that only start local commands can use it too. Needs [uv](https://docs.astral.sh/uv/).
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
uvx llmbenchmark-mcp
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Claude Desktop (`claude_desktop_config.json`):
|
|
74
|
+
|
|
75
|
+
```json
|
|
76
|
+
{
|
|
77
|
+
"mcpServers": {
|
|
78
|
+
"llmbenchmark": {
|
|
79
|
+
"command": "uvx",
|
|
80
|
+
"args": ["llmbenchmark-mcp"]
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Docker, built from this repository:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
docker build -t llmbenchmark-mcp . && docker run -i --rm llmbenchmark-mcp
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Set `LLMBENCHMARK_MCP_URL` to point the proxy at another endpoint.
|
|
93
|
+
|
|
94
|
+
## Tools
|
|
95
|
+
|
|
96
|
+
| Tool | What it does |
|
|
97
|
+
|---|---|
|
|
98
|
+
| `recommend_model` | Task text plus optional limits (`max_output_price_per_m`, `min_context_tokens`, `open_weights_only`, `top_k`) in, ranked models with scores, prices and a one-line reason out. |
|
|
99
|
+
| `get_leaderboard` | Top models on one board: overall, coding, agentic, reasoning, math, knowledge, design. |
|
|
100
|
+
| `compare_models` | Two models side by side on every board where both were measured. |
|
|
101
|
+
| `get_model` | One model: maker, prices, context, open weights, score and rank per board. |
|
|
102
|
+
| `get_prices` | API list prices per million tokens, cheapest first, by model or maker. |
|
|
103
|
+
| `list_categories` | The categories, the board behind each and the task keywords that select it. |
|
|
104
|
+
|
|
105
|
+
`recommend_model` maps the task to a category with a fixed keyword map (printed at https://llmbenchmark.ai/api#how-recommend-works), drops models that break your limits and ranks the rest on that category's board. If nothing fits your limits it says so and returns the nearest options.
|
|
106
|
+
|
|
107
|
+
## Example prompts
|
|
108
|
+
|
|
109
|
+
- "Which model should I use to refactor a React app under $5 per million output tokens?"
|
|
110
|
+
- "What is the best open-weight model for math with at least a 128k context window?"
|
|
111
|
+
- "Compare Claude Opus and GPT-6 Astra for coding."
|
|
112
|
+
- "Which models are cheapest on output price from DeepSeek?"
|
|
113
|
+
|
|
114
|
+
## CLI
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
llmbenchmark-mcp recommend "refactor a React app" --max-price 5 --open-weights
|
|
118
|
+
llmbenchmark-mcp tools
|
|
119
|
+
llmbenchmark-mcp config
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
`recommend` prints the `recommend_model` answer, `tools` lists the tool names, `config` prints the remote JSON config for Claude or Cursor. With no arguments the command runs the stdio server.
|
|
123
|
+
|
|
124
|
+
## REST
|
|
125
|
+
|
|
126
|
+
The same answers over plain HTTP:
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
curl "https://api.llmbenchmark.ai/v1/recommend?task=refactor+a+large+React+codebase&max_output_price_per_m=5"
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## License
|
|
133
|
+
|
|
134
|
+
- This README and the configuration files in this repository: MIT, see `LICENSE`.
|
|
135
|
+
- The data the server returns: CC BY 4.0. Credit "LLMBenchmark.ai" with a link to https://llmbenchmark.ai. Each score names its source board and the day it was read; the boards keep their own terms.
|
|
136
|
+
|
|
137
|
+
Run by [Take Group](https://takegroup.pl).
|
|
138
|
+
|
|
139
|
+
<!-- mcp-name: ai.llmbenchmark/llmbenchmark -->
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# LLMBenchmark.ai MCP server
|
|
2
|
+
|
|
3
|
+
Ask "which model should I use for this task?" and get an answer from benchmark scores and API prices of 400+ LLMs, updated daily.
|
|
4
|
+
|
|
5
|
+
- Free. No API key. 500 requests per IP per UTC day.
|
|
6
|
+
- Remote server, Streamable HTTP: `https://api.llmbenchmark.ai/mcp`
|
|
7
|
+
- Deterministic: no LLM call inside, the same task and the same data return the same answer.
|
|
8
|
+
- Every answer carries the data date, the source ("LLMBenchmark.ai") and links to the pages behind the numbers.
|
|
9
|
+
|
|
10
|
+
Docs: https://llmbenchmark.ai/api
|
|
11
|
+
|
|
12
|
+
## Install
|
|
13
|
+
|
|
14
|
+
**Claude Code**
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
claude mcp add --transport http llmbenchmark https://api.llmbenchmark.ai/mcp
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
**Cursor** (`.cursor/mcp.json`)
|
|
21
|
+
|
|
22
|
+
```json
|
|
23
|
+
{
|
|
24
|
+
"mcpServers": {
|
|
25
|
+
"llmbenchmark": { "url": "https://api.llmbenchmark.ai/mcp" }
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
**Codex** (`~/.codex/config.toml`)
|
|
31
|
+
|
|
32
|
+
```toml
|
|
33
|
+
[mcp_servers.llmbenchmark]
|
|
34
|
+
url = "https://api.llmbenchmark.ai/mcp"
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
**Claude Desktop** (`claude_desktop_config.json`, through the `mcp-remote` bridge)
|
|
38
|
+
|
|
39
|
+
```json
|
|
40
|
+
{
|
|
41
|
+
"mcpServers": {
|
|
42
|
+
"llmbenchmark": {
|
|
43
|
+
"command": "npx",
|
|
44
|
+
"args": ["-y", "mcp-remote", "https://api.llmbenchmark.ai/mcp"]
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
**Install as a local server (stdio)**
|
|
51
|
+
|
|
52
|
+
The package proxies to the hosted server, so clients that only start local commands can use it too. Needs [uv](https://docs.astral.sh/uv/).
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
uvx llmbenchmark-mcp
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Claude Desktop (`claude_desktop_config.json`):
|
|
59
|
+
|
|
60
|
+
```json
|
|
61
|
+
{
|
|
62
|
+
"mcpServers": {
|
|
63
|
+
"llmbenchmark": {
|
|
64
|
+
"command": "uvx",
|
|
65
|
+
"args": ["llmbenchmark-mcp"]
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Docker, built from this repository:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
docker build -t llmbenchmark-mcp . && docker run -i --rm llmbenchmark-mcp
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Set `LLMBENCHMARK_MCP_URL` to point the proxy at another endpoint.
|
|
78
|
+
|
|
79
|
+
## Tools
|
|
80
|
+
|
|
81
|
+
| Tool | What it does |
|
|
82
|
+
|---|---|
|
|
83
|
+
| `recommend_model` | Task text plus optional limits (`max_output_price_per_m`, `min_context_tokens`, `open_weights_only`, `top_k`) in, ranked models with scores, prices and a one-line reason out. |
|
|
84
|
+
| `get_leaderboard` | Top models on one board: overall, coding, agentic, reasoning, math, knowledge, design. |
|
|
85
|
+
| `compare_models` | Two models side by side on every board where both were measured. |
|
|
86
|
+
| `get_model` | One model: maker, prices, context, open weights, score and rank per board. |
|
|
87
|
+
| `get_prices` | API list prices per million tokens, cheapest first, by model or maker. |
|
|
88
|
+
| `list_categories` | The categories, the board behind each and the task keywords that select it. |
|
|
89
|
+
|
|
90
|
+
`recommend_model` maps the task to a category with a fixed keyword map (printed at https://llmbenchmark.ai/api#how-recommend-works), drops models that break your limits and ranks the rest on that category's board. If nothing fits your limits it says so and returns the nearest options.
|
|
91
|
+
|
|
92
|
+
## Example prompts
|
|
93
|
+
|
|
94
|
+
- "Which model should I use to refactor a React app under $5 per million output tokens?"
|
|
95
|
+
- "What is the best open-weight model for math with at least a 128k context window?"
|
|
96
|
+
- "Compare Claude Opus and GPT-6 Astra for coding."
|
|
97
|
+
- "Which models are cheapest on output price from DeepSeek?"
|
|
98
|
+
|
|
99
|
+
## CLI
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
llmbenchmark-mcp recommend "refactor a React app" --max-price 5 --open-weights
|
|
103
|
+
llmbenchmark-mcp tools
|
|
104
|
+
llmbenchmark-mcp config
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
`recommend` prints the `recommend_model` answer, `tools` lists the tool names, `config` prints the remote JSON config for Claude or Cursor. With no arguments the command runs the stdio server.
|
|
108
|
+
|
|
109
|
+
## REST
|
|
110
|
+
|
|
111
|
+
The same answers over plain HTTP:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
curl "https://api.llmbenchmark.ai/v1/recommend?task=refactor+a+large+React+codebase&max_output_price_per_m=5"
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
## License
|
|
118
|
+
|
|
119
|
+
- This README and the configuration files in this repository: MIT, see `LICENSE`.
|
|
120
|
+
- The data the server returns: CC BY 4.0. Credit "LLMBenchmark.ai" with a link to https://llmbenchmark.ai. Each score names its source board and the day it was read; the boards keep their own terms.
|
|
121
|
+
|
|
122
|
+
Run by [Take Group](https://takegroup.pl).
|
|
123
|
+
|
|
124
|
+
<!-- mcp-name: ai.llmbenchmark/llmbenchmark -->
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Example prompts
|
|
2
|
+
|
|
3
|
+
Ask these in any client with the LLMBenchmark.ai server connected.
|
|
4
|
+
|
|
5
|
+
- "Which model should I use to refactor a React app under $5 per million output tokens?"
|
|
6
|
+
- "What is the best open-weight model for math with at least a 128k context window?"
|
|
7
|
+
- "Compare Claude Opus and GPT-6 Astra for coding."
|
|
8
|
+
- "Which models are cheapest on output price from DeepSeek?"
|
|
9
|
+
- "Which model is best for agentic tool use with a 200k context window?"
|
|
10
|
+
- "Show the top 10 models on the reasoning leaderboard."
|
|
11
|
+
- "What does Gemini cost per million input and output tokens compared with Claude Sonnet?"
|
|
12
|
+
- "Which categories can you rank models on, and which task keywords pick each one?"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"$schema": "https://glama.ai/mcp/schemas/server.json", "maintainers": ["slugc"]}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""LLMBenchmark.ai MCP server: stdio proxy and small CLI over the hosted server (stdlib only)."""
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
import urllib.error
|
|
8
|
+
import urllib.request
|
|
9
|
+
|
|
10
|
+
VERSION = "1.0.0"
|
|
11
|
+
URL = os.environ.get("LLMBENCHMARK_MCP_URL") or "https://api.llmbenchmark.ai/mcp"
|
|
12
|
+
PROTOCOLS = ("2025-06-18", "2025-03-26", "2024-11-05")
|
|
13
|
+
USER_AGENT = "llmbenchmark-mcp/" + VERSION
|
|
14
|
+
|
|
15
|
+
CONFIG = {"mcpServers": {"llmbenchmark": {"url": "https://api.llmbenchmark.ai/mcp"}}}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Client:
|
|
19
|
+
"""Talks to the hosted Streamable HTTP server; handles JSON and SSE replies and the session id."""
|
|
20
|
+
|
|
21
|
+
def __init__(self):
|
|
22
|
+
self.session = None
|
|
23
|
+
self.next_id = 0
|
|
24
|
+
self.started = False
|
|
25
|
+
self.tools = None
|
|
26
|
+
|
|
27
|
+
def _post(self, payload):
|
|
28
|
+
headers = {
|
|
29
|
+
"Accept": "application/json, text/event-stream",
|
|
30
|
+
"Content-Type": "application/json",
|
|
31
|
+
"User-Agent": USER_AGENT,
|
|
32
|
+
}
|
|
33
|
+
if self.session:
|
|
34
|
+
headers["Mcp-Session-Id"] = self.session
|
|
35
|
+
req = urllib.request.Request(URL, json.dumps(payload).encode(), headers, method="POST")
|
|
36
|
+
try:
|
|
37
|
+
with urllib.request.urlopen(req, timeout=60) as resp:
|
|
38
|
+
self.session = resp.headers.get("Mcp-Session-Id") or self.session
|
|
39
|
+
return resp.read().decode(), resp.headers.get("Content-Type", "")
|
|
40
|
+
except urllib.error.HTTPError as e:
|
|
41
|
+
raise RuntimeError("HTTP %s: %s" % (e.code, e.read().decode(errors="replace")[:300]))
|
|
42
|
+
except urllib.error.URLError as e:
|
|
43
|
+
raise RuntimeError("Cannot reach %s: %s" % (URL, e.reason))
|
|
44
|
+
|
|
45
|
+
@staticmethod
|
|
46
|
+
def _parse(body, ctype, rid):
|
|
47
|
+
if "text/event-stream" in ctype:
|
|
48
|
+
for line in body.splitlines():
|
|
49
|
+
if line.startswith("data:"):
|
|
50
|
+
try:
|
|
51
|
+
msg = json.loads(line[5:].strip())
|
|
52
|
+
except ValueError:
|
|
53
|
+
continue
|
|
54
|
+
if isinstance(msg, dict) and msg.get("id") == rid:
|
|
55
|
+
return msg
|
|
56
|
+
raise RuntimeError("No JSON-RPC answer in stream")
|
|
57
|
+
return json.loads(body)
|
|
58
|
+
|
|
59
|
+
def rpc(self, method, params=None):
|
|
60
|
+
self.next_id += 1
|
|
61
|
+
rid = self.next_id
|
|
62
|
+
body, ctype = self._post({"jsonrpc": "2.0", "id": rid, "method": method, "params": params or {}})
|
|
63
|
+
msg = self._parse(body, ctype, rid)
|
|
64
|
+
if "error" in msg:
|
|
65
|
+
raise RuntimeError(msg["error"].get("message", str(msg["error"])))
|
|
66
|
+
return msg.get("result", {})
|
|
67
|
+
|
|
68
|
+
def start(self):
|
|
69
|
+
if self.started:
|
|
70
|
+
return
|
|
71
|
+
self.rpc("initialize", {
|
|
72
|
+
"protocolVersion": PROTOCOLS[0],
|
|
73
|
+
"capabilities": {},
|
|
74
|
+
"clientInfo": {"name": "llmbenchmark-mcp", "version": VERSION},
|
|
75
|
+
})
|
|
76
|
+
self._post({"jsonrpc": "2.0", "method": "notifications/initialized"})
|
|
77
|
+
self.started = True
|
|
78
|
+
|
|
79
|
+
def list_tools(self):
|
|
80
|
+
if self.tools is None:
|
|
81
|
+
self.start()
|
|
82
|
+
self.tools = self.rpc("tools/list").get("tools", [])
|
|
83
|
+
return self.tools
|
|
84
|
+
|
|
85
|
+
def call(self, name, args):
|
|
86
|
+
self.start()
|
|
87
|
+
return self.rpc("tools/call", {"name": name, "arguments": args or {}})
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def result_text(res):
|
|
91
|
+
return "\n".join(c.get("text", "") for c in res.get("content", []) if c.get("type") == "text")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class Server:
|
|
95
|
+
def __init__(self):
|
|
96
|
+
self.client = Client()
|
|
97
|
+
|
|
98
|
+
def handle(self, msg):
|
|
99
|
+
method, rid, params = msg.get("method"), msg.get("id"), msg.get("params") or {}
|
|
100
|
+
if rid is None:
|
|
101
|
+
return None # notification, no reply
|
|
102
|
+
if method == "initialize":
|
|
103
|
+
want = params.get("protocolVersion")
|
|
104
|
+
result = {
|
|
105
|
+
"protocolVersion": want if want in PROTOCOLS else PROTOCOLS[0],
|
|
106
|
+
"capabilities": {"tools": {"listChanged": False}},
|
|
107
|
+
"serverInfo": {"name": "llmbenchmark", "title": "LLMBenchmark.ai", "version": VERSION},
|
|
108
|
+
}
|
|
109
|
+
elif method == "ping":
|
|
110
|
+
result = {}
|
|
111
|
+
elif method == "tools/list":
|
|
112
|
+
try:
|
|
113
|
+
result = {"tools": self.client.list_tools()}
|
|
114
|
+
except Exception as e:
|
|
115
|
+
return {"jsonrpc": "2.0", "id": rid, "error": {"code": -32603, "message": str(e)}}
|
|
116
|
+
elif method == "tools/call":
|
|
117
|
+
try:
|
|
118
|
+
result = self.client.call(params.get("name", ""), params.get("arguments"))
|
|
119
|
+
except Exception as e:
|
|
120
|
+
self.client = Client()
|
|
121
|
+
result = {"content": [{"type": "text", "text": "LLMBenchmark.ai is unreachable: %s" % e}], "isError": True}
|
|
122
|
+
else:
|
|
123
|
+
return {"jsonrpc": "2.0", "id": rid, "error": {"code": -32601, "message": "Method not found: %s" % method}}
|
|
124
|
+
return {"jsonrpc": "2.0", "id": rid, "result": result}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def serve():
|
|
128
|
+
"""MCP stdio transport: one JSON-RPC message per line on stdin/stdout, logs on stderr."""
|
|
129
|
+
server = Server()
|
|
130
|
+
for line in sys.stdin:
|
|
131
|
+
line = line.strip()
|
|
132
|
+
if not line:
|
|
133
|
+
continue
|
|
134
|
+
msg = None
|
|
135
|
+
try:
|
|
136
|
+
msg = json.loads(line)
|
|
137
|
+
reply = server.handle(msg) if isinstance(msg, dict) else None
|
|
138
|
+
except ValueError:
|
|
139
|
+
reply = {"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "Parse error"}}
|
|
140
|
+
except Exception as e:
|
|
141
|
+
print("Error: %s" % e, file=sys.stderr)
|
|
142
|
+
reply = {"jsonrpc": "2.0", "id": msg.get("id") if isinstance(msg, dict) else None,
|
|
143
|
+
"error": {"code": -32603, "message": str(e)}}
|
|
144
|
+
if reply is not None:
|
|
145
|
+
sys.stdout.write(json.dumps(reply) + "\n")
|
|
146
|
+
sys.stdout.flush()
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def main():
|
|
150
|
+
p = argparse.ArgumentParser(prog="llmbenchmark-mcp", description="LLMBenchmark.ai MCP server and CLI. No args: stdio server.")
|
|
151
|
+
p.add_argument("--version", action="version", version="llmbenchmark-mcp " + VERSION)
|
|
152
|
+
sub = p.add_subparsers(dest="cmd")
|
|
153
|
+
r = sub.add_parser("recommend", help="print the recommend_model answer for a task")
|
|
154
|
+
r.add_argument("task")
|
|
155
|
+
r.add_argument("--max-price", type=float, help="max output price per million tokens")
|
|
156
|
+
r.add_argument("--open-weights", action="store_true", help="open-weight models only")
|
|
157
|
+
sub.add_parser("tools", help="print the tool names")
|
|
158
|
+
sub.add_parser("config", help="print the remote JSON config for Claude/Cursor")
|
|
159
|
+
args = p.parse_args()
|
|
160
|
+
if args.cmd is None:
|
|
161
|
+
return serve()
|
|
162
|
+
if args.cmd == "config":
|
|
163
|
+
print(json.dumps(CONFIG, indent=2))
|
|
164
|
+
return
|
|
165
|
+
try:
|
|
166
|
+
c = Client()
|
|
167
|
+
if args.cmd == "tools":
|
|
168
|
+
print("\n".join(t["name"] for t in c.list_tools()))
|
|
169
|
+
else:
|
|
170
|
+
a = {"task": args.task}
|
|
171
|
+
if args.max_price is not None:
|
|
172
|
+
a["max_output_price_per_m"] = args.max_price
|
|
173
|
+
if args.open_weights:
|
|
174
|
+
a["open_weights_only"] = True
|
|
175
|
+
res = c.call("recommend_model", a)
|
|
176
|
+
print(result_text(res))
|
|
177
|
+
if res.get("isError"):
|
|
178
|
+
sys.exit(1)
|
|
179
|
+
except Exception as e:
|
|
180
|
+
print("Error: %s" % e, file=sys.stderr)
|
|
181
|
+
sys.exit(1)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
if __name__ == "__main__":
|
|
185
|
+
main()
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "llmbenchmark-mcp"
|
|
3
|
+
version = "1.0.0"
|
|
4
|
+
description = "Which LLM for which task: benchmark scores and API prices for 400+ models as an MCP server. Free, no API key."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.10"
|
|
9
|
+
authors = [{ name = "LLMBenchmark.ai", email = "hello@llmbenchmark.ai" }]
|
|
10
|
+
keywords = ["llm benchmark", "llm leaderboard", "llm api pricing", "mcp server", "claude", "chatgpt", "cursor", "model comparison"]
|
|
11
|
+
classifiers = ["Programming Language :: Python :: 3", "Topic :: Software Development"]
|
|
12
|
+
dependencies = []
|
|
13
|
+
|
|
14
|
+
[project.urls]
|
|
15
|
+
Homepage = "https://llmbenchmark.ai/api?utm_source=pypi&utm_medium=package&utm_campaign=mcp"
|
|
16
|
+
Source = "https://github.com/Take-Group/llmbenchmark-mcp"
|
|
17
|
+
|
|
18
|
+
[project.scripts]
|
|
19
|
+
llmbenchmark-mcp = "llmbenchmark_mcp:main"
|
|
20
|
+
|
|
21
|
+
[build-system]
|
|
22
|
+
requires = ["hatchling>=1.27"]
|
|
23
|
+
build-backend = "hatchling.build"
|
|
24
|
+
|
|
25
|
+
[tool.hatch.build.targets.wheel.force-include]
|
|
26
|
+
"llmbenchmark_mcp.py" = "llmbenchmark_mcp.py"
|
|
27
|
+
|
|
28
|
+
[tool.hatch.build.targets.sdist]
|
|
29
|
+
include = ["llmbenchmark_mcp.py", "README.md", "LICENSE", "mcp.json", "Dockerfile", "glama.json", "server.json", "examples", ".cursor"]
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
|
|
3
|
+
"name": "ai.llmbenchmark/llmbenchmark",
|
|
4
|
+
"title": "LLMBenchmark.ai",
|
|
5
|
+
"description": "Which LLM for which task: benchmark scores and API prices for 400+ models. Free, no key.",
|
|
6
|
+
"version": "1.0.0",
|
|
7
|
+
"websiteUrl": "https://llmbenchmark.ai/api",
|
|
8
|
+
"repository": {
|
|
9
|
+
"url": "https://github.com/Take-Group/llmbenchmark-mcp",
|
|
10
|
+
"source": "github"
|
|
11
|
+
},
|
|
12
|
+
"remotes": [
|
|
13
|
+
{
|
|
14
|
+
"type": "streamable-http",
|
|
15
|
+
"url": "https://api.llmbenchmark.ai/mcp"
|
|
16
|
+
}
|
|
17
|
+
],
|
|
18
|
+
"icons": [
|
|
19
|
+
{
|
|
20
|
+
"src": "https://llmbenchmark.ai/apple-touch-icon.png",
|
|
21
|
+
"mimeType": "image/png",
|
|
22
|
+
"sizes": [
|
|
23
|
+
"180x180"
|
|
24
|
+
]
|
|
25
|
+
}
|
|
26
|
+
]
|
|
27
|
+
}
|