premanmcp 1.1.7 → 1.1.9

package/bin/eval_harness.py

@@ -1,6 +1,6 @@
-"""Run assert-ai against Anthropic, working around four upstream gaps.
+"""Run assert-ai against Anthropic, working around five upstream gaps.
 
-All four workarounds live here rather than in the installed package so they
+All five workarounds live here rather than in the installed package so they
 survive `uv sync`, which re-downloads the pinned ASSERT rev and discards any
 edit made under .venv/.
 
@@ -48,12 +48,39 @@ edit made under .venv/.
    exactly the object the prompt describes -- so the value was always meant to
    be that document, serialised.
 
-All four go away upstream: (1) with a workspace-scoped API key, (2), (3) and
-(4) with a fix in AleWang16/ASSERT plus a bumped rev in pyproject.toml.
+5. A callable target is tool-blind to the judge. ``CallableSession.run_turn``
+   builds ``interaction_messages`` containing only ``user`` and ``assistant``
+   roles, and the inference stage emits a ``ToolCallEdit`` -- the row the judge
+   reads and ``missingToolEvidence`` counts -- only from a ``role: "tool"``
+   message. So every tool call a callable target makes is invisible to scoring,
+   silently: a tool-defined behaviour scores as "the agent called nothing".
+
+   ``HTTPEndpointSession`` does not have this problem, because it routes its
+   reply through ``_normalize_connector_response`` and
+   ``_serialize_connector_interaction_messages``, and the latter maps an
+   adapter's ``tool_result`` event to ``role: "tool"``. The two target kinds
+   disagree about whether tool calls exist.
+
+   So this workaround gives the callable path the endpoint path's treatment: a
+   callable returning an adapter-shaped ``{"response": str, "events": [...]}``
+   is routed through those same two helpers. That makes a local shim's return
+   value mean exactly what the same JSON means over HTTP, which is what lets an
+   agent be evaluated in-process without changing what its scores mean.
+
+   ``run_turn`` is replaced rather than wrapped. The obvious wrapper -- stash
+   the raw dict on the instance, let the original normalise -- is not safe: the
+   inference stage fans out concurrently over one session object, so anything
+   kept on ``self`` between the call and its use can be overwritten by another
+   turn. A replacement holds the payload in a local.
+
+Workarounds (1)-(4) go away upstream: (1) with a workspace-scoped API key, (2),
+(3) and (4) with a fix in AleWang16/ASSERT plus a bumped rev in pyproject.toml.
+(5) goes away when ``CallableSession`` serialises tool evidence the way
+``HTTPEndpointSession`` already does.
 
 This file exists twice. ``preman-mcp/bin/eval_harness.py`` is a byte-identical
 copy, shipped in the npm package so a run leased to a paired device goes through
-the same three workarounds a server-side run does -- otherwise an Anthropic
+the same workarounds a server-side run does -- otherwise an Anthropic
 tester would silently lose the multi-turn half of every bank on a customer's
 machine and keep it on ours, and the two runs would not be comparable while
 looking like they were. ``tests/test_eval_harness_shim.py`` fails if the copies
@@ -229,6 +256,140 @@ def _build_chat_payload_without_extra_body(model, messages, options):
 # at call time, so rebinding here reaches all of them.
 model_client._build_chat_payload = _build_chat_payload_without_extra_body
 
+
+from assert_ai.core import session as _session
+from assert_ai.core.async_utils import invoke_callable as _invoke_callable
+from assert_ai.core.model_client import _classify_llm_error
+from assert_ai.core.session import CallableSession, Message, TurnResult
+
+
+def _is_adapter_payload(value) -> bool:
+    """Whether a callable returned the endpoint contract rather than a reply.
+
+    ``response`` is the discriminator because it is the required key of that
+    contract and is not a key assert-ai's own normaliser looks for: it reads
+    ``text``, ``content`` and ``choices``. A dict carrying ``response`` would
+    otherwise fall through to ``str(result)`` and be scored as the repr of a
+    dict, so claiming it here takes nothing from the paths that already work.
+    """
+    return isinstance(value, dict) and isinstance(value.get("response"), str)
+
+
+async def _callable_run_turn(self, messages: list[Message]) -> TurnResult:
+    user_text = ""
+    for msg in reversed(messages):
+        if msg.role == "user":
+            user_text = msg.text
+            break
+
+    kwargs = {"timeout_s": self._message_timeout_s}
+    if self._supports_history:
+        kwargs["history"] = [
+            {"role": msg.role, "content": msg.text}
+            for msg in messages
+            if msg.role in ("user", "assistant")
+        ]
+
+    try:
+        raw_result = await _invoke_callable(self._callable, user_text, **kwargs)
+    except Exception as exc:
+        # Upstream's reasoning, preserved: a user callable makes its own
+        # provider calls, so a content-filter rejection arrives as a raw
+        # litellm error rather than a typed LLM*Error, and the inference
+        # stage's per-seed isolation keys off the typed ones. Anything the
+        # classifier does not recognise is a genuine crash and propagates.
+        classified = _classify_llm_error(exc)
+        if classified is exc:
+            raise
+        raise classified from exc
+
+    if not _is_adapter_payload(raw_result):
+        # Upstream's own two paths, inlined because the original `run_turn`
+        # invokes the callable itself and so cannot be handed a result that
+        # has already been produced. Calling it would run the agent twice.
+        result = self._normalize_callable_result(raw_result)
+        if isinstance(result, _session.ModelResponse):
+            response_text = result.text or ""
+            interaction_messages = [
+                {"role": "user", "content": user_text},
+                {"role": "assistant", "content": response_text},
+            ]
+            if result.tool_calls:
+                interaction_messages[1]["tool_calls"] = [
+                    tc.to_openai_dict() for tc in result.tool_calls
+                ]
+            return TurnResult(
+                text=response_text,
+                state_messages=list(messages) + [result.message],
+                interaction_messages=interaction_messages,
+                tool_traces=[
+                    _session.ToolTrace(
+                        tool_name=tc.name, tool_args=tc.arguments, tool_result=""
+                    )
+                    for tc in (result.tool_calls or [])
+                ],
+                llm_calls=[_session.build_llm_call_trace(result, source="callable")],
+                raw={
+                    "callable": self._callable_ref,
+                    "model": result.model,
+                    "usage": _session.summarize_response(result).get("usage"),
+                },
+            )
+        return TurnResult(
+            text=result,
+            state_messages=list(messages) + [Message(role="assistant", content=result)],
+            interaction_messages=[
+                {"role": "user", "content": user_text},
+                {"role": "assistant", "content": result},
+            ],
+            raw={"callable": self._callable_ref},
+        )
+
+    # Sanitised on the same grounds the endpoint path gives: event content is
+    # agent-influenced and lands in run artifacts, so it needs the redaction
+    # the response text gets.
+    response = _session._normalize_connector_response({
+        "text": _session._sanitize_response_text(raw_result["response"]),
+        "events": _session._sanitize_endpoint_value(raw_result.get("events")),
+        "raw": {"callable": self._callable_ref},
+    })
+    interaction_messages = _session._serialize_connector_interaction_messages(
+        user_text=user_text,
+        response=response,
+    )
+    return TurnResult(
+        text=response.text,
+        state_messages=list(messages) + [Message(role="assistant", content=response.text)],
+        interaction_messages=interaction_messages,
+        tool_traces=[
+            _session.ToolTrace(
+                tool_name=event.tool_name or "tool",
+                tool_args=event.tool_args or {},
+                tool_result=event.content,
+            )
+            for event in (response.events or [])
+            if event.role == "tool_result"
+        ],
+        raw=response.raw,
+    )
+
+
+CallableSession.run_turn = _callable_run_turn
+
+# The customer's agent lives in their repository, not next to this file, and
+# assert-ai resolves a `callable:` target only as a dotted module name -- there
+# is no spec key that adds an import root, and `_temporary_sys_path` only ever
+# covers the config directory and the cwd, neither of which is the repo. The
+# run's cwd cannot move either: progress and artifacts are read relative to it.
+#
+# So the import root arrives out of band. A PreMan-specific variable rather
+# than PYTHONPATH because the interpreter may be `uv run --project`, and uv's
+# handling of PYTHONPATH is not ours to depend on; plain inheritance is.
+for _root in os.environ.get("PREMAN_EVAL_SYS_PATH", "").split(os.pathsep):
+    _root = _root.strip()
+    if _root and os.path.isdir(_root) and _root not in sys.path:
+        sys.path.insert(0, _root)
+
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
 _cli = next(e for e in entry_points(group="console_scripts") if e.name == "assert-ai")

package/bin/eval_runtime.js

@@ -0,0 +1,156 @@
+/**
+ * PreMan's own Python for running an eval, fetched once and kept.
+ *
+ * The problem this solves: an eval executes on the customer's machine, so that
+ * machine needs an interpreter that can `import assert_ai`. Until now the CLI
+ * borrowed one, and which one it borrowed depended on the working directory —
+ * `uv run --project <cwd>` when a `pyproject.toml` happened to be there, else
+ * whatever `python3` resolved to. A customer pasting the dashboard's command
+ * into a Node or Go repo matches neither and is refused; worse, a `python3` that
+ * can import `assert_ai` but cannot complete a run is silently preferred over a
+ * known-good one. That happened: a run picked up a conda interpreter, spent five
+ * and a half minutes, wrote no artifacts, and reported `ok`.
+ *
+ * So this is a runtime that belongs to PreMan rather than to the customer's
+ * project, keyed by the assert-ai revision it contains and cached under
+ * `~/.preman/runtime`. Fetched on first use because the user asked for an eval,
+ * which is also the whole of the consent: nothing is installed into their Python
+ * or their project, and nothing is written outside `~/.preman`.
+ *
+ * Two hard rules, both about not being worse than doing nothing:
+ *
+ *   - Never throw. Every failure path returns `""` so the caller falls back to
+ *     the interpreters it already knew about. A machine that already has a
+ *     working assert-ai must not be blocked by a proxy or a bad digest.
+ *   - Never leave a partial tree where a whole one is expected. The extract
+ *     lands in a temp directory and is renamed into place, because the cache
+ *     check is "does the interpreter exist" and a half-unpacked bundle would
+ *     pass it.
+ *
+ * Deliberately *not* the right interpreter for every run. A `callable` target
+ * has the harness import the customer's own agent in-process, and this bundle
+ * cannot import their dependencies — see `resolveInterpreter` in eval.js, which
+ * is where that choice is made. See docs/EVAL_SELF_CONTAINED_RUNTIME.md.
+ */
+
+import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+/** Everything PreMan keeps that is not one customer's data. */
+const RUNTIME_HOME = path.join(os.homedir(), ".preman", "runtime");
+
+/**
+ * The assert-ai revision this bundle contains, truncated.
+ *
+ * Part of the cache path rather than a version file beside it, so changing the
+ * pin fetches a new tree next to the old one instead of mutating one in place —
+ * and so a run can never use a bundle that does not match what this CLI expects.
+ */
+export const RUNTIME_PIN = "14f4e2136ab7";
+
+/** sha256 of the bundle, pinned here rather than fetched beside it.
+ *
+ * A digest served from the same host as the artifact proves only that the host
+ * is internally consistent. Pinning it in the source that was published through
+ * npm means the two would have to be compromised together.
+ */
+export const RUNTIME_SHA256 =
+  "f92becee5252262594326b533bd86a4e8b6a4a97080af0626804eb49c4e7222c";
+
+const RUNTIME_BASE =
+  process.env.PREMAN_EVAL_RUNTIME_BASE ||
+  "https://github.com/PreMan-Inc/PreMan-Desktop/releases/download/eval-runtime-1";
+
+/** The same rule `suiteHome` applies, for the same reason: this becomes a directory name. */
+const SAFE_SEGMENT = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
+
+/**
+ * Which bundle this machine can run, or `""` for one we do not build.
+ *
+ * Proof of concept: Linux x64 only. Everything else falls through to the
+ * interpreters the CLI already knew how to find, with no network call — a
+ * Windows or macOS user must not pay a download to learn there isn't one.
+ */
+export function runtimePlatform({ platform = process.platform, arch = process.arch } = {}) {
+  return platform === "linux" && arch === "x64" ? "linux-x64" : "";
+}
+
+/** Where this machine's interpreter lives, whether or not it is there yet. */
+export function runtimeInterpreter({ pin = RUNTIME_PIN, platform = runtimePlatform(), home = RUNTIME_HOME } = {}) {
+  if (!platform || !SAFE_SEGMENT.test(pin) || !SAFE_SEGMENT.test(platform)) return "";
+  return path.join(home, pin, platform, "python", "bin", "python3");
+}
+
+function sha256Hex(file) {
+  return createHash("sha256").update(readFileSync(file)).digest("hex");
+}
+
+/**
+ * The cached interpreter, fetching it first if this is the first run.
+ *
+ * Returns a path, or `""` — never throws, and never the path to something it
+ * could not finish installing.
+ */
+export async function ensureRuntime({
+  log = () => {},
+  home = RUNTIME_HOME,
+  pin = RUNTIME_PIN,
+  digest = RUNTIME_SHA256,
+  base = RUNTIME_BASE,
+  download,
+  progress,
+} = {}) {
+  const platform = runtimePlatform();
+  if (!platform) return "";
+
+  const target = runtimeInterpreter({ pin, platform, home });
+  if (!target) return "";
+  if (existsSync(target)) return target;
+  if (!download) return "";
+
+  const name = `preman-eval-runtime-${pin}-${platform}.tar.gz`;
+  const url = `${base}/${name}`;
+  let work = "";
+  const bar = progress ? progress() : { tick: () => {}, clear: () => {} };
+  try {
+    mkdirSync(home, { recursive: true, mode: 0o700 });
+    work = mkdtempSync(path.join(home, ".fetch-"));
+    const archive = path.join(work, name);
+
+    log(`Preparing the eval runtime — first run only`);
+    await download(url, archive, bar.tick);
+    bar.clear();
+
+    // Refuse rather than run something we cannot identify. An unset digest is a
+    // build that was never pinned, which is a mistake to surface, not to ignore.
+    if (!digest) throw new Error("this build has no runtime digest pinned, so the download cannot be verified");
+    const got = sha256Hex(archive);
+    if (got !== digest) throw new Error(`digest mismatch: expected ${digest}, got ${got}`);
+
+    const unpacked = path.join(work, "tree");
+    mkdirSync(unpacked, { recursive: true, mode: 0o700 });
+    const tar = spawnSync("tar", ["xzf", archive, "-C", unpacked], { stdio: ["ignore", "ignore", "pipe"], encoding: "utf8" });
+    if (tar.status !== 0) throw new Error(`could not unpack the runtime: ${(tar.stderr || "").trim() || tar.status}`);
+
+    const slot = path.join(home, pin, platform);
+    mkdirSync(path.dirname(slot), { recursive: true, mode: 0o700 });
+    // Last, and atomic: until this lands there is nothing at the path the cache
+    // check looks at, so an interrupted fetch leaves a miss rather than a ruin.
+    renameSync(unpacked, slot);
+
+    if (!existsSync(target)) throw new Error(`the runtime unpacked without an interpreter at ${target}`);
+    log(`  eval runtime ready (${path.join(home, pin, platform)})`);
+    return target;
+  } catch (error) {
+    bar.clear();
+    // Logged, not thrown. The caller has other interpreters to try and this
+    // machine may well have a perfectly good one.
+    log(`  could not prepare the eval runtime (${error.message}); looking for a Python already here`);
+    return "";
+  } finally {
+    if (work) rmSync(work, { recursive: true, force: true });
+  }
+}