ninjaeval-mcp 0.1.0__py3-none-any.whl

ninjaeval_mcp/__init__.py

@@ -0,0 +1,2 @@
+"""NinjaEval MCP server: test an AI change from a coding agent before calling it done."""
+__version__ = "0.1.0"

ninjaeval_mcp/api.py

@@ -0,0 +1,234 @@
+"""NinjaEval API client and the compact summaries the MCP tools return. Standard library only, so it can be tested
+without the MCP package."""
+import json
+import os
+import ssl
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+DEFAULT_API = "https://api.ninjaeval.com/api/v1/eng"
+DEFAULT_APP = "https://ninjaeval.com"
+PASS = "PASS"
+FAILING = ("regressed", "errored")
+
+
+def _tls():
+    """certifi's CA bundle when installed (mcp brings it via httpx): python.org builds on macOS have none by default."""
+    try:
+        import certifi
+        return ssl.create_default_context(cafile=certifi.where())
+    except ImportError:
+        return ssl.create_default_context()
+
+
+class NinjaEvalError(RuntimeError):
+    pass
+
+
+def _key_from_env():
+    key = os.environ.get("NINJAEVAL_API_KEY", "").strip()
+    path = os.environ.get("NINJAEVAL_API_KEY_FILE", "").strip()
+    if not key and path:
+        with open(os.path.expanduser(path)) as f:
+            key = f.read().strip()
+    return key
+
+
+class Client:
+    """One workspace, through its API key (`efk_…`, from ⚙ Workspace & API keys)."""
+
+    def __init__(self, key=None, api=None, app=None, sleep=time.sleep, clock=time.monotonic):
+        self.key = key if key is not None else _key_from_env()
+        self.api = (api or os.environ.get("NINJAEVAL_API_URL") or DEFAULT_API).rstrip("/")
+        self.app = (app or os.environ.get("NINJAEVAL_APP_URL") or DEFAULT_APP).rstrip("/")
+        self.sleep, self.clock = sleep, clock
+
+    # ── HTTP ──
+    def call(self, method, path, body=None, text=False):
+        if not self.key:
+            raise NinjaEvalError("No API key: set NINJAEVAL_API_KEY (or NINJAEVAL_API_KEY_FILE) to a workspace key "
+                                 "from NinjaEval → ⚙ Workspace & API keys.")
+        req = urllib.request.Request(self.api + path, method=method,
+                                     data=json.dumps(body).encode() if body is not None else None,
+                                     headers={"Authorization": f"Bearer {self.key}", "Content-Type": "application/json",
+                                              "User-Agent": "ninjaeval-mcp"})
+        try:
+            with urllib.request.urlopen(req, timeout=60, context=_tls()) as r:
+                raw = r.read().decode()
+        except urllib.error.HTTPError as e:
+            detail = e.read().decode(errors="replace")
+            try:
+                detail = json.loads(detail).get("detail", detail)
+            except ValueError:
+                pass
+            raise NinjaEvalError(f"NinjaEval API {method} {path}: {e.code} {detail}") from None
+        except urllib.error.URLError as e:
+            raise NinjaEvalError(f"Can't reach NinjaEval at {self.api}: {e.reason}") from None
+        return raw if text else (json.loads(raw) if raw else None)
+
+    # ── reads ──
+    def me(self):
+        m = self.call("GET", "/me")
+        return {k: m.get(k) for k in ("workspace", "role", "can_run", "can_manage", "workspace_limits") if k in m}
+
+    def projects(self):
+        return [{"project_id": p["project_id"], "name": p.get("name"), "run_mode": p.get("run_mode"),
+                 "description": p.get("description") or ""} for p in self.call("GET", "/projects")["projects"]]
+
+    def project(self, project_id):
+        v = self.call("GET", f"/projects/{q(project_id)}")
+        prod = v.get("production") or {}
+        return {
+            "project_id": project_id, "name": v["project"].get("name"), "run_mode": v["project"].get("run_mode"),
+            "gate": v["project"].get("gate"),
+            "production": {k: prod.get(k) for k in ("label", "provider", "model", "params", "system_prompt", "tools", "code_sha", "note")}
+            if prod else None,
+            "suites": [{"dataset_id": d["dataset_id"], "name": d.get("name"), "cases": d.get("item_count")} for d in v.get("datasets") or []],
+            "ready_to_test": all((v.get("setup") or {}).values()),
+            "setup": v.get("setup"),
+            "recent_change_tests": [{"test_id": t["test_id"], "candidate": t.get("candidate_label"), "baseline": t.get("baseline_label"),
+                                     "status": t.get("status"), "complete": t.get("complete"), "created_at": t.get("created_at"),
+                                     "url": self.test_url(t["test_id"])} for t in (v.get("change_tests") or [])[:5]],
+        }
+
+    def test(self, test_id):
+        return self.call("GET", f"/change-tests/{q(test_id)}")
+
+    def report(self, test_id):
+        return self.call("GET", f"/change-tests/{q(test_id)}/report.md", text=True)
+
+    def test_url(self, test_id):
+        return f"{self.app}/#/change/{test_id}"
+
+    # ── writes ──
+    def start_change_test(self, project_id, **changes):
+        body = {k: v for k, v in changes.items() if v is not None}
+        return self.call("POST", f"/projects/{q(project_id)}/test-change", body)
+
+    def analyze(self, test_id, item_ids, runs=3):
+        return self.call("POST", f"/change-tests/{q(test_id)}/analyze", {"item_ids": list(item_ids), "runs": runs})
+
+    # ── waiting ──
+    def wait(self, test_id, timeout=600, poll=5, diag_grace=120, item_ids=None):
+        """Poll until the runs finish and the diagnoses of the failing cases (or `item_ids`) settle, or the timeout.
+        A diagnosis that hasn't started `diag_grace` seconds after the runs finish isn't waited for."""
+        start = self.clock()
+        finished_at = None
+        while True:
+            t = self.test(test_id)
+            if runs_done(t):
+                finished_at = finished_at if finished_at is not None else self.clock()
+                waiting = diagnoses_pending(t, item_ids)
+                if not waiting or (not any(waiting.values()) and self.clock() - finished_at >= diag_grace):
+                    return t
+            if self.clock() - start >= timeout:
+                return t
+            self.sleep(poll)
+
+
+def q(s):
+    return urllib.parse.quote(str(s), safe="")
+
+
+def runs_done(t):
+    p = t.get("progress") or {}
+    return bool(p.get("total")) and p.get("done") == p.get("total") and (t.get("gate") or {}).get("verdict") != "pending"
+
+
+def diagnoses_pending(t, item_ids=None):
+    """{item_id: started?} for the cases whose diagnosis is still to come."""
+    want = set(item_ids) if item_ids else None
+    out = {}
+    for r in t.get("rows") or []:
+        if want is not None:
+            if r["item_id"] not in want:
+                continue
+        elif r.get("kind") != "regressed":
+            continue
+        rca = r.get("rca") or {}
+        if rca.get("status") not in ("done", "error"):
+            out[r["item_id"]] = bool(rca)
+    return out
+
+
+def _clip(s, n=1500):
+    s = s or ""
+    return s if len(s) <= n else s[: n - 1] + "…"
+
+
+def diagnosis(rca):
+    if not rca:
+        return None
+    if rca.get("status") != "done":
+        return {"status": rca.get("status") or "queued", **({"error": _clip(rca.get("error"), 300)} if rca.get("error") else {})}
+    res = rca.get("result") or {}
+    d, fix, ver = res.get("diagnosis") or {}, res.get("fix") or {}, res.get("verification") or None
+    out = {"status": "done", "cause": d.get("cause"), "kind": d.get("kind"), "explanation": _clip(d.get("explanation")),
+           "agreement": res.get("agreement"), "needs_review": bool(res.get("needs_review")),
+           "fix": {k: fix[k] for k in ("type", "summary", "prompt_edits", "test_fix", "risky") if fix.get(k)}}
+    if ver:
+        out["fix_verified"] = {"status": ver.get("status"), "note": _clip(ver.get("note"), 400)}
+    prior = res.get("prior") or {}
+    if prior.get("flagged"):
+        out["prior_disagrees"] = {"cause": prior.get("cause"), "note": "a confirmed past diagnosis of a similar failure says otherwise; check before acting"}
+    return out
+
+
+def _side(s):
+    return {"label": s.get("label"), "score": s.get("score")} if s else None
+
+
+def summary(t, url):
+    """What a coding agent needs from a change test: the verdict, why, and what to do next."""
+    gate, test = t.get("gate") or {}, t.get("test") or {}
+    rows = t.get("rows") or []
+    done = runs_done(t)
+    bs, cs = t.get("baseline_summary") or {}, t.get("candidate_summary") or {}
+    num = lambda s: {k: s.get(k) for k in ("passed", "cases", "avg_score") if k in s}
+    failing = [r for r in rows if r.get("kind") in FAILING]
+    still_failing = [r for r in rows if r.get("kind") in ("same", "worse") and (r.get("candidate") or {}).get("label") not in (None, PASS)]
+    out = {
+        "test_id": test.get("test_id"), "project_id": test.get("project_id"), "url": url,
+        "status": "done" if done else "running",
+        "progress": f"{(t.get('progress') or {}).get('done', 0)}/{(t.get('progress') or {}).get('total', 0)} answers",
+        "verdict": gate.get("verdict"),
+        **({"would_be": gate["would_be"]} if gate.get("would_be") else {}),
+        **({"stage": gate["stage"]} if gate.get("stage") else {}),
+        "rules": [{"rule": r.get("rule"), "passed": r.get("passed"), "detail": r.get("detail")} for r in gate.get("rules") or []],
+        "counts": t.get("counts"),
+        "baseline": {"label": (t.get("baseline") or {}).get("label"), **num(bs)},
+        "candidate": {"label": (t.get("candidate") or {}).get("label"), "note": (t.get("candidate") or {}).get("note"), **num(cs)},
+        "regressions": [{"item_id": r["item_id"], "title": r.get("title"), "kind": r["kind"], "tags": r.get("tags") or [],
+                         "before": _side(r.get("baseline")), "after": _side(r.get("candidate")),
+                         "diagnosis": diagnosis(r.get("rca"))} for r in failing],
+        "still_failing": [{"item_id": r["item_id"], "title": r.get("title"), "kind": r["kind"], "after": _side(r.get("candidate")),
+                           **({"new_failures": r["new_failures"]} if r.get("new_failures") else {}),
+                           **({"diagnosis": diagnosis(r["rca"])} if r.get("rca") else {})} for r in still_failing],
+    }
+    out["next"] = next_step(out)
+    return out
+
+
+def next_step(s):
+    if s["status"] != "done":
+        return f"Still running ({s['progress']}). Call get_change_test with test_id {s['test_id']} and wait=true."
+    v = s["verdict"]
+    if v == "ready":
+        return ("The gate passes. Shipping (promoting to production) is a person's decision: share the url, or let CI "
+                "run the same test on the pull request.")
+    diags = [r["diagnosis"] for r in s["regressions"] if r.get("diagnosis")]
+    if any(d.get("status") != "done" for d in diags) or len(diags) < len(s["regressions"]):
+        return "Diagnoses are still running; call get_change_test with wait=true to read the causes and fixes."
+    if any(d.get("cause") == "TEST" for d in diags):
+        hint = " Where the cause is TEST, the test itself is wrong: tell the user rather than changing the product to fit it."
+    else:
+        hint = ""
+    if any((d.get("fix") or {}).get("prompt_edits") for d in diags):
+        return ("Blocked. Apply each regression's prompt_edits (find → replace) to the system prompt, or write a better fix, "
+                "then call test_change again. Don't remove or weaken test cases to pass the gate." + hint)
+    if v in ("blocked", "partial"):
+        return ("Blocked. Read each regression's diagnosis and fix, change the prompt, model, tools or code, and call "
+                "test_change again. Don't remove or weaken test cases to pass the gate." + hint)
+    return f"Verdict: {v}."

ninjaeval_mcp/server.py

@@ -0,0 +1,116 @@
+"""NinjaEval MCP server: a coding agent tests an AI change (prompt, model, tools, code) before calling it done.
+
+Runs over stdio; it opens no network port. It acts as one workspace, through that workspace's API key.
+It can start change tests and diagnoses but can't ship: promoting a version to production stays with a person.
+"""
+import os
+from typing import List, Optional
+
+try:                                                     # mcp 2.x
+    from mcp.server.mcpserver import MCPServer as _Server
+except ImportError:                                      # mcp 1.x
+    from mcp.server.fastmcp import FastMCP as _Server
+
+from .api import Client, NinjaEvalError, summary
+
+mcp = _Server("ninjaeval", instructions=(
+    "NinjaEval is a release gate for AI changes. After you change an AI app's system prompt, model, tools or agent code, "
+    "call test_change: it replays the project's test cases on production and on your change, grades every answer, and "
+    "returns a ready / blocked verdict with the cause and a fix for each regression. Don't call a prompt or model change "
+    "done while the verdict is blocked. Each test runs the real app and an AI judge, so it costs money and takes minutes: "
+    "test once per meaningful change, not after every keystroke. Never delete or weaken test cases to pass the gate."))
+
+_client = None
+_started = 0
+MAX_TESTS = int(os.environ.get("NINJAEVAL_MCP_MAX_TESTS", "10"))      # per server session, against an agent looping on fixes
+
+
+def client():
+    global _client
+    if _client is None:
+        _client = Client()
+    return _client
+
+
+@mcp.tool()
+def whoami() -> dict:
+    """The workspace this server acts in, and what its API key may do."""
+    return client().me()
+
+
+@mcp.tool()
+def list_projects() -> list:
+    """The workspace's projects (AI apps under test). Use project_id in the other tools."""
+    return client().projects()
+
+
+@mcp.tool()
+def get_project(project_id: str) -> dict:
+    """A project's production version (provider, model, system prompt, tools), release gate, test suites and recent
+    change tests. Read this before test_change to see what production is."""
+    return client().project(project_id)
+
+
+@mcp.tool()
+def test_change(project_id: str, system_prompt: Optional[str] = None, model: Optional[str] = None,
+                provider: Optional[str] = None, params: Optional[dict] = None, tools: Optional[List[dict]] = None,
+                note: str = "", code_sha: str = "", stage: Optional[str] = None, trials: int = 1,
+                wait: bool = True, timeout_seconds: int = 600) -> dict:
+    """Test a change against production. Copies production and applies only the fields you pass: system_prompt (the
+    full new prompt), model, provider, params, tools ([{name, description, parameters}]). For an endpoint project whose
+    change is in code, deploy it where the project's endpoint runs it and pass code_sha. note says what changed and why.
+    stage runs only that stage's cases (e.g. "smoke"; quick, can't ship). trials > 1 runs each case several times.
+
+    With wait=true (default) it returns when the verdict and the regressions' diagnoses are in, or at timeout_seconds
+    (then call get_change_test). Returns verdict (ready / blocked / partial / pending), the gate's rules, each regression
+    with its diagnosis (cause PROMPT / MODEL / TOOL / TEST, explanation, fix with prompt_edits), a url and a next step."""
+    global _started
+    if _started >= MAX_TESTS:
+        raise NinjaEvalError(f"This session already started {MAX_TESTS} change tests (NINJAEVAL_MCP_MAX_TESTS). "
+                             "Stop and show the user the results so far.")
+    c = client()
+    t = c.start_change_test(project_id, system_prompt=system_prompt, model=model, provider=provider, params=params,
+                            tools=tools, note=note or "Change from a coding agent (MCP)", code_sha=code_sha, stage=stage,
+                            trials=max(1, min(int(trials), 5)))
+    _started += 1
+    test_id = t["test_id"]
+    full = c.wait(test_id, timeout=_timeout(timeout_seconds)) if wait else c.test(test_id)
+    return summary(full, c.test_url(test_id))
+
+
+@mcp.tool()
+def get_change_test(test_id: str, wait: bool = False, timeout_seconds: int = 600) -> dict:
+    """A change test's verdict, rules, regressions with their diagnoses and fixes, and the next step. With wait=true,
+    first waits for the runs and the diagnoses to finish (up to timeout_seconds)."""
+    c = client()
+    full = c.wait(test_id, timeout=_timeout(timeout_seconds)) if wait else c.test(test_id)
+    return summary(full, c.test_url(test_id))
+
+
+@mcp.tool()
+def diagnose_cases(test_id: str, item_ids: List[str], wait: bool = True, timeout_seconds: int = 300) -> dict:
+    """Diagnose failing cases that weren't diagnosed automatically (regressions are). item_ids come from
+    get_change_test's still_failing. Each diagnosis runs 3 times and re-runs its fix to check it."""
+    c = client()
+    c.analyze(test_id, item_ids)
+    full = c.wait(test_id, timeout=_timeout(timeout_seconds), item_ids=item_ids) if wait else c.test(test_id)
+    return summary(full, c.test_url(test_id))
+
+
+@mcp.tool()
+def get_report(test_id: str) -> str:
+    """The change test's full report as Markdown: both versions, every case result, the gate and the diagnoses.
+    Good for a pull request description."""
+    return client().report(test_id)
+
+
+def _timeout(s):
+    return max(10, min(int(s), 1800))
+
+
+def main():
+    mcp.run("stdio")       # stdio only: no network listener
+
+
+if __name__ == "__main__":
+    main()

ninjaeval_mcp-0.1.0.dist-info/METADATA

@@ -0,0 +1,71 @@
+Metadata-Version: 2.4
+Name: ninjaeval-mcp
+Version: 0.1.0
+Summary: NinjaEval as an MCP server: a coding agent tests a prompt, model or tool change and reads the verdict and the diagnosis before calling it done.
+License: MIT
+Project-URL: Homepage, https://ninjaeval.com
+Project-URL: Documentation, https://ninjaeval.com/docs/developer-guide.html
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: mcp<3,>=1.2
+Dynamic: license-file
+
+# NinjaEval MCP server
+
+Lets your coding agent (Claude Code, Cursor, Copilot, any MCP client) test an AI change with
+[NinjaEval](https://ninjaeval.com) before it calls the change done.
+
+The agent edits your bot's system prompt, model or tools, then calls `test_change`. NinjaEval replays your test cases
+on production and on the change and grades every answer. It returns **ready** or **blocked**, with the cause of each
+regression and a fix that was re-run to check it. The agent applies the fix and tests again. Shipping stays with you:
+the server can't promote a version to production.
+
+## Set up
+
+1. In NinjaEval, open **⚙ Workspace & API keys** and create a key for the agent (e.g. `coding-agent`).
+2. Add the server to your MCP client. For Claude Code:
+
+   ```bash
+   claude mcp add ninjaeval --env NINJAEVAL_API_KEY=efk_... -- uvx ninjaeval-mcp
+   ```
+
+   Any other client (`mcpServers` in its JSON config):
+
+   ```json
+   { "mcpServers": { "ninjaeval": { "command": "uvx", "args": ["ninjaeval-mcp"],
+                                    "env": { "NINJAEVAL_API_KEY": "efk_..." } } } }
+   ```
+
+   Or `pip install ninjaeval-mcp` and run `ninjaeval-mcp`.
+
+It runs over stdio on your machine and opens no network port. It talks only to `api.ninjaeval.com`.
+
+## Tools
+
+| Tool | What it does |
+|---|---|
+| `whoami` | The workspace, the key's role, today's limits |
+| `list_projects` | Your projects (AI apps under test) |
+| `get_project` | Production's provider, model, system prompt and tools; the gate; suites; recent change tests |
+| `test_change` | Tests a change against production: only the fields you pass change (`system_prompt`, `model`, `provider`, `params`, `tools`, `code_sha`, `note`, `stage`, `trials`). Waits for the verdict and the diagnoses |
+| `get_change_test` | A test's verdict, rules, regressions with cause / explanation / fix (`prompt_edits`) / whether the fix was verified, cases still failing, and the next step |
+| `diagnose_cases` | Diagnoses failing cases that weren't diagnosed automatically |
+| `get_report` | The full report as Markdown, e.g. for a PR description |
+
+Each response has a `next` field telling the agent what to do. When the cause is `TEST`, the test itself is wrong, so
+the agent should tell you rather than bend the product to fit it.
+
+## Settings
+
+| Variable | Default | |
+|---|---|---|
+| `NINJAEVAL_API_KEY` | | Workspace key (`efk_…`) |
+| `NINJAEVAL_API_KEY_FILE` | | Read the key from a file instead |
+| `NINJAEVAL_API_URL` | `https://api.ninjaeval.com/api/v1/eng` | |
+| `NINJAEVAL_APP_URL` | `https://ninjaeval.com` | Used for the links in responses |
+| `NINJAEVAL_MCP_MAX_TESTS` | `10` | Change tests one session may start, so an agent can't loop on fixes indefinitely |
+
+Each change test runs your real app and an AI judge, which takes minutes and counts toward the workspace's daily
+limits. Endpoint projects call your service with the system prompt NinjaEval sends. If your service ignores it, test
+code changes by deploying them where the endpoint runs and passing `code_sha`.

ninjaeval_mcp-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+ninjaeval_mcp/__init__.py,sha256=v0WybslcsUSZFOdphi8QbZP7KzjcMWHCUxUqXuc5qiU,112
+ninjaeval_mcp/api.py,sha256=Ijo4ZwNYb5WPelLd4_7zWzQKodn-tCvVumN8EnWdQ1Q,11505
+ninjaeval_mcp/server.py,sha256=Y2L3L5q2Xsg6wHq8ESxfFLm7tywh5wgYEeJaL7RWmWw,5562
+ninjaeval_mcp-0.1.0.dist-info/licenses/LICENSE,sha256=fHJAF0hijeS_sAkdlfykh85X_0241uZohegGFxSbCW0,1066
+ninjaeval_mcp-0.1.0.dist-info/METADATA,sha256=-tlbKxzpllalcjICCYmg4jxPoYoYraH07d5zcd5JeTw,3520
+ninjaeval_mcp-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+ninjaeval_mcp-0.1.0.dist-info/entry_points.txt,sha256=aIwwm6onhWLrF1GwX_vNNSWMR3IzztaZrJf6TrIehgw,60
+ninjaeval_mcp-0.1.0.dist-info/top_level.txt,sha256=d3MfGEisXJnv4BA7P1nSfxDYaAvPdXgsobhzqkAdTbQ,14
+ninjaeval_mcp-0.1.0.dist-info/RECORD,,

ninjaeval_mcp-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

ninjaeval_mcp-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+ninjaeval-mcp = ninjaeval_mcp.server:main

ninjaeval_mcp-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 NinjaEval
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ninjaeval_mcp-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+ninjaeval_mcp