premanmcp 1.1.11 → 1.1.13

package/bin/eval.js

@@ -238,7 +238,7 @@ function tidy(home) {
  * assumption every path-traversal bug is built on. A frame that names
  * `../../.ssh/authorized_keys` is refused here rather than written.
  */
-export function materialize(job, root) {
+export function materialize(job, root, { trace = false } = {}) {
   mkdirSync(root, { recursive: true, mode: 0o700 });
   const bundle = path.join(root, "bundle");
   // Emptied first, because the directory outlives the run now. Every file the
@@ -259,13 +259,65 @@ export function materialize(job, root) {
     writeFileSync(target, String(entry.text ?? ""), { mode: 0o600 });
   }
 
+  const spec = JSON.parse(JSON.stringify(job.spec ?? {}));
+  // Traced only when this device asked for it and the target is the in-process
+  // callable: assert-ai then uses OTelTracedSession, which switches the
+  // installed plugins on before importing the agent. A spec that already says
+  // how to trace keeps its own setting.
+  const traceTarget = spec.pipeline?.inference?.target;
+  if (trace && traceTarget && typeof traceTarget === "object" && traceTarget.callable && !traceTarget.trace) {
+    traceTarget.trace = { backend: "otel", group_by: "session.id" };
+  }
+  const taxonomy = spec._preman_generation_taxonomy;
+  const hasSavedTaxonomy = taxonomy !== null && typeof taxonomy === "object" && !Array.isArray(taxonomy);
+  delete spec._preman_generation_taxonomy;
+
+  // A dispatched bank is already selected, including an explicitly empty one.
+  // Generating cases again would spend money and replace the questions being rerun.
+  if (job.case_bank != null) {
+    if (!Array.isArray(job.case_bank)) throw new EvalError("the run carried an invalid case bank");
+    const outputDir = path.resolve(runDir(root, job));
+    if (!outputDir.startsWith(path.resolve(root) + path.sep)) {
+      throw new EvalError("refusing to write outside the run directory: case_map.json");
+    }
+    const bankPath = path.resolve(bundle, "preman-selected-cases.jsonl");
+    writeFileSync(bankPath, job.case_bank.map((row) => `${JSON.stringify(row)}\n`).join(""), { mode: 0o600 });
+    const pipeline = spec.pipeline ||= {};
+    delete pipeline.test_set;
+    // Legacy subsets still need a rubric when no saved taxonomy arrived.
+    if (hasSavedTaxonomy) delete pipeline.systematize;
+    const inference = pipeline.inference ||= {};
+    inference.test_set_path = bankPath;
+
+    // ASSERT renumbers its input by position. Store the translation alongside
+    // the run's results so a subset still answers its original bank cases.
+    const mapping = Object.fromEntries(job.case_bank.flatMap((row, index) =>
+      typeof row?.test_case_id === "string" && row.test_case_id
+        ? [[`test_case_${String(index + 1).padStart(6, "0")}`, row.test_case_id]]
+        : []
+    ));
+    if (Object.keys(mapping).length) {
+      mkdirSync(outputDir, { recursive: true, mode: 0o700 });
+      writeFileSync(path.join(outputDir, "case_map.json"), `${JSON.stringify(mapping, null, 2)}\n`, { mode: 0o600 });
+    }
+
+    // Curated jobs already reference a bundled rubric. Only replace it when
+    // dispatch supplies the rubric captured with a generated bank.
+    const judge = pipeline.judge;
+    if (hasSavedTaxonomy && judge && typeof judge === "object" && !Array.isArray(judge)) {
+      const taxonomyPath = path.resolve(bundle, "preman-generation-taxonomy.json");
+      writeFileSync(taxonomyPath, `${JSON.stringify(taxonomy, null, 2)}\n`, { mode: 0o600 });
+      judge.taxonomy_path = taxonomyPath;
+    }
+  }
+
   // The spec is written as JSON with a .yaml name, which is legal and not a
   // shortcut: YAML is a superset of JSON, assert-ai's loader accepts it, and
   // JSON.stringify cannot mis-quote a value the way a hand-rolled YAML emitter
   // can. Shipping a YAML serializer to avoid a filename that looks odd would be
   // the worse trade.
   const configPath = path.join(bundle, "preman-eval.yaml");
-  writeFileSync(configPath, `${JSON.stringify(job.spec ?? {}, null, 2)}\n`, { mode: 0o600 });
+  writeFileSync(configPath, `${JSON.stringify(spec, null, 2)}\n`, { mode: 0o600 });
   return { cwd: root, bundle, configPath };
 }
 
@@ -298,6 +350,35 @@ export const HARNESS_SPEC =
   String(process.env.PREMAN_EVAL_HARNESS_SPEC || "").trim() ||
   "git+https://github.com/AleWang16/ASSERT@14f4e2136ab753897ce4cb90613e1663730aeb70";
 
+/**
+ * OpenInference plugins added to the environment when a run is traced
+ * (`PREMAN_EVAL_TRACE=1`, docs/EVAL_AGENT_TRACING.md). Each one only records
+ * calls into a library the agent actually imports.
+ *
+ * Two tiers, because a framework plugin can pin its framework's version and a
+ * customer's own pin may disagree: the resolve is tried with everything, then
+ * with the provider SDKs alone, then with none. A plugin must never be the
+ * reason a run cannot start.
+ */
+export const TRACE_PROVIDER_PLUGINS = [
+  "openinference-instrumentation-openai",
+  "openinference-instrumentation-anthropic",
+  "openinference-instrumentation-google-genai",
+  "openinference-instrumentation-bedrock",
+];
+export const TRACE_FRAMEWORK_PLUGINS = [
+  "openinference-instrumentation-langchain",
+  "openinference-instrumentation-llama-index",
+  "openinference-instrumentation-openai-agents",
+];
+
+/** Shapes whose agent runs inside the harness's own Python process. */
+const IN_PROCESS_SHAPES = new Set(["py_history", "py_plain", "py_class", "langgraph", "other", ""]);
+
+export function tracesInProcess(target) {
+  return target?.kind === "callable" && target.language === "python" && IN_PROCESS_SHAPES.has(target.shape || "");
+}
+
 /** How long a first resolve may take before it counts as a failure. */
 const COLD_RESOLVE_MS = 600_000;
 
@@ -410,15 +491,29 @@ export function resolveInterpreter({ projectPath = process.cwd(), log = () => {}
     // fetches the harness's whole dependency tree, and treating a slow network
     // as "this interpreter cannot import assert_ai" would send somebody looking
     // for a missing package.
-    const added = ["run", "--project", projectPath, "--with", HARNESS_SPEC, "python"];
-    const resolved = probeInterpreter("uv", added, { timeoutMs: COLD_RESOLVE_MS });
-    if (resolved) {
-      return {
-        command: "uv",
-        argv: added,
-        version: resolved,
-        how: "uv run in this project, with PreMan's harness alongside it",
-      };
+    const tiers =
+      process.env.PREMAN_EVAL_TRACE === "1"
+        ? [
+            { plugins: [...TRACE_PROVIDER_PLUGINS, ...TRACE_FRAMEWORK_PLUGINS], note: ", tracing the agent's model and framework calls" },
+            { plugins: TRACE_PROVIDER_PLUGINS, note: ", tracing the agent's model calls (framework plugins did not resolve against this project)" },
+            { plugins: [], note: " (tracing requested, but no tracing plugin resolved against this project)" },
+          ]
+        : [{ plugins: [], note: "" }];
+    for (const { plugins, note } of tiers) {
+      const added = [
+        "run", "--project", projectPath, "--with", HARNESS_SPEC,
+        ...plugins.flatMap((plugin) => ["--with", plugin]),
+        "python",
+      ];
+      const resolved = probeInterpreter("uv", added, { timeoutMs: COLD_RESOLVE_MS });
+      if (resolved) {
+        return {
+          command: "uv",
+          argv: added,
+          version: resolved,
+          how: `uv run in this project, with PreMan's harness alongside it${note}`,
+        };
+      }
     }
   }
 
@@ -637,9 +732,10 @@ export const RUN_ARTIFACTS = [
  * No scope is sent with an upload. The server derives it from the filename
  * (`eval_runner_artifacts.scope_of`), files these under the suite rather than
  * the run, and applies its own write-once rule — so a second run of the same
- * suite offers them again and the server declines to rewrite history.
+ * suite offers them again and the server declines to rewrite history. The
+ * generation report is mutable and describes jobs that did not enter the bank.
  */
-export const SUITE_ARTIFACTS = ["taxonomy.json", "test_set.jsonl"];
+export const SUITE_ARTIFACTS = ["taxonomy.json", "test_set.jsonl", "generation.json"];
 
 /**
  * Upload the artifacts that changed since last time.
@@ -1168,7 +1264,7 @@ export async function executeEvalRun(
   try {
     home = suiteHome(job);
     pruneRuns(home, job);
-    laid = materialize(job, home);
+    laid = materialize(job, home, { trace: process.env.PREMAN_EVAL_TRACE === "1" });
     // Offered only when the harness is the only thing that has to import
     // anything. A `callable` target imports the customer's agent in-process, so
     // it must resolve against an environment that has *their* dependencies —
@@ -1553,11 +1649,24 @@ export async function summarizeRuns(args, key, runs, { say = () => {} } = {}) {
 
 export const EVAL_HELP = `
 Eval options:
+  eval                          Measure your agent. With no flags this is a
+                                conversation: PreMan asks what the agent does
+                                and narrows to the two or three behaviours worth
+                                measuring, rather than pricing everything that
+                                happens to be switched on
+  eval --only <id>              Skip the conversation and measure one behaviour
+  --yes                         Do not ask before spending. For CI
+  --fail-on <rate>              Exit 1 if the pass rate is below this. Accepts
+                                0.8 or 80. Off by default, so adding this to a
+                                pipeline cannot fail a build that passed before
+  --detach                      Start it and hand the terminal back
   eval run --token <pmev_...>   Redeem a token from the dashboard: find the agent
                                 on this machine and run exactly the evals the
                                 token names
   eval run                      Pair this project and run the eval PreMan has
-                                queued for it, then exit
+                                queued for it, then exit. Also \`runner work\`,
+                                which says what it is: this is the machine
+                                volunteering, not you asking
   eval target                   Find the code in this repository that runs your
                                 agent, write a wrapper the eval can call, and
                                 remember it once you approve it
@@ -1671,42 +1780,77 @@ export function watchRunTerminal(args, runId, { token, intervalMs = 3_000, fetch
 
 async function runForThisAccount(args, deps) {
   const say = (line) => process.stdout.write(`${line}\n`);
-
-  const key = resolveApiKey(args);
-  if (!key) {
-    throw new EvalError(
-      `no PreMan API key on this machine, so there is no account to read the ` +
-        `switched-on behaviours from. Run \`${cliInvocation()} login\`.`
-    );
+  const {
+    chooseBehaviors,
+    confirmSpend,
+    quoteRun,
+    requireKey,
+    EXIT_DID_NOT_RUN,
+  } = await import("./eval_start.js");
+
+  let key;
+  let chosen;
+  try {
+    key = requireKey(args);
+    chosen = await chooseBehaviors(args, key, { only: args.value("--only", "") });
+  } catch (error) {
+    throw new EvalError(error.message);
   }
 
-  const catalog = await callBackendJson(args, "GET", "/eval-behaviors", { token: key });
-  if (!catalog.ok) {
-    throw new EvalError(
-      `could not read what this account measures (${catalog.status_code}): ` +
-        `${describeFailure(catalog)}`
+  // Detaching changes who runs it, which is why it changes which check applies.
+  //
+  // A foreground run executes in this process: the harness, the adapter and the
+  // agent are all here, and the terminal staying open is what keeps them alive.
+  // There is nothing to hand it to on the way out, so `--detach` cannot mean
+  // "the same run, without the wait" -- it has to mean "queue this to the
+  // machine that is already connected and listening". That is a different
+  // destination, and the account's four readiness states are exactly the
+  // question of whether it exists.
+  const detach = args.has("--detach");
+  if (detach) {
+    const { accountReadiness, reportNotReady, EXIT_DID_NOT_RUN } = await import(
+      "./eval_start.js"
     );
+    const state = await accountReadiness(args, key);
+    if (state.state !== "ready") {
+      say(
+        "--detach queues the run to a connected machine rather than running it " +
+          "here, and there is not one yet."
+      );
+      say("");
+      reportNotReady(state, { say });
+      return { ok: false, reason: state.state, exitCode: EXIT_DID_NOT_RUN };
+    }
   }
 
-  const enabled = (catalog.behaviors || []).filter((entry) => entry.enabled);
-  const only = args.value("--only", "").trim();
-  const chosen = only
-    ? enabled.filter((entry) => entry.id === only || entry.name === only)
-    : enabled;
-
-  if (only && chosen.length === 0) {
-    // Naming the ones that are on beats "not found": the usual cause is a
-    // behaviour that exists but is switched off, and that is a different fix
-    // from a typo.
-    const names = enabled.map((entry) => entry.id).join(", ") || "nothing";
-    throw new EvalError(`${only} is not switched on. Currently on: ${names}`);
-  }
-  if (chosen.length === 0) {
-    throw new EvalError(
-      "nothing is switched on for this account, so there is nothing to measure. " +
-        "Turn a behaviour on under Agent eval → Settings, or describe one in your " +
-        "own words there."
-    );
+  // The agent has to be findable before anybody is quoted for measuring it.
+  //
+  // Deliberately not the account's four readiness states, which answer "could
+  // the server dispatch this to a connected daemon". A foreground run *is* the
+  // runner for its own duration, so gating it on a daemon would refuse a local
+  // run for the absence of something it does not use. The honest local question
+  // is whether there is an approved entry point here, and `allowSearch: false`
+  // asks it without writing a wrapper or prompting: the search belongs to
+  // `eval target`, on purpose, with somebody watching.
+  if (
+    !detach &&
+    !String(args.value("--target", "") || args.value("--preman-agent", "")).trim()
+  ) {
+    let found = null;
+    try {
+      found = await discoverTarget(args, { log: () => {}, allowSearch: false });
+    } catch {
+      found = null;
+    }
+    if (!found) {
+      say(
+        "No agent has been approved in this repository yet, so there is nothing " +
+          "here to measure. Find one once and approve it:"
+      );
+      say("");
+      say(`  ${cliInvocation()} eval target`);
+      return { ok: false, reason: "no_target", exitCode: EXIT_DID_NOT_RUN };
+    }
   }
 
   // One run each, and said plainly before anything is spent: assert-ai's
@@ -1715,6 +1859,37 @@ async function runForThisAccount(args, deps) {
   say(`${chosen.length} behaviour${chosen.length === 1 ? "" : "s"} switched on:`);
   for (const entry of chosen) say(`  ${entry.title || entry.id}`);
 
+  const quote = await quoteRun(args, key, {
+    behaviorIds: chosen.map((entry) => entry.id),
+    endpoint: String(args.value("--endpoint", "") || "").trim(),
+  });
+  const agreed = await confirmSpend(quote, {
+    assumeYes: args.has("--yes"),
+    say,
+  });
+  if (!agreed) {
+    say("Nothing started, nothing charged.");
+    return { ok: false, reason: "declined", exitCode: EXIT_DID_NOT_RUN };
+  }
+
+  if (detach) {
+    const queued = await callBackendJson(args, "POST", "/eval-sessions/dispatch", {
+      token: key,
+      json: { behavior_ids: chosen.map((entry) => entry.id) },
+    });
+    if (!queued.ok) {
+      throw new EvalError(
+        `could not queue (${queued.status_code}): ${describeFailure(queued)}`
+      );
+    }
+    const runs = queued.runs || [];
+    say(`Queued ${runs.length} run${runs.length === 1 ? "" : "s"} on ${queued.target?.label || "the connected machine"}.`);
+    for (const row of runs) say(`  ${row.id}`);
+    say("");
+    say("The result appears in the PreMan app when it finishes.");
+    return { ok: true, detached: true, runs, exitCode: 0 };
+  }
+
   const opened = await callBackendJson(args, "POST", "/eval-sessions", {
     token: key,
     json: { behavior_ids: chosen.map((entry) => entry.id) },
@@ -1725,7 +1900,44 @@ async function runForThisAccount(args, deps) {
     );
   }
 
-  return runWithToken(opened.token, args, deps);
+  const finished = await runWithToken(opened.token, args, deps);
+
+  const { exitCodeFor } = await import("./eval_start.js");
+  const totals = tallyCases(finished.summaries);
+  return {
+    ...finished,
+    totals,
+    exitCode: exitCodeFor(totals, { failOn: args.value("--fail-on", "") }),
+  };
+}
+
+/**
+ * Cases and passes across every behaviour, for a gate to compare.
+ *
+ * Totalled rather than averaged: a behaviour of 2 cases and one of 20 are not
+ * equal votes, and one behaviour passing perfectly does not offset another that
+ * failed.
+ *
+ * A `flagged` case is one the judge marked against the rubric, so it is not a
+ * pass. Neither is `not_run` -- a case that never reached the agent told us
+ * nothing, and counting it either way would be inventing a result. It is
+ * counted in the denominator so that a run which mostly failed to execute
+ * cannot report a perfect rate off the handful that did.
+ */
+export function tallyCases(summaries = []) {
+  return (summaries || []).reduce(
+    (sum, entry) => {
+      const row = entry?.row ?? entry ?? {};
+      const scored = Number(row.scored) || 0;
+      const flagged = Number(row.flagged) || 0;
+      const notRun = Number(row.not_run) || 0;
+      return {
+        cases: sum.cases + scored + notRun,
+        passed: sum.passed + Math.max(0, scored - flagged),
+      };
+    },
+    { cases: 0, passed: 0 }
+  );
 }
 
 /**
@@ -1776,6 +1988,17 @@ async function runWithToken(
     // import path, so the harness reads this and inserts it before handing off.
     process.env.PREMAN_EVAL_SYS_PATH = target.sysPath;
   }
+  // On unless PREMAN_EVAL_TRACE=0 (docs/EVAL_AGENT_TRACING.md). Kept only for an
+  // agent that runs inside the harness's Python process; a node bridge or a
+  // subprocess is invisible to in-process tracing, and tracing it would only
+  // replace the adapter's reported tool calls with nothing.
+  const traceAsked = process.env.PREMAN_EVAL_TRACE;
+  if (traceAsked !== "0") {
+    process.env.PREMAN_EVAL_TRACE = tracesInProcess(target) ? "1" : "0";
+    if (traceAsked === "1" && !tracesInProcess(target)) {
+      say(`tracing is not available for this agent (${target.language || "?"} / ${target.shape || "?"}); running untraced`);
+    }
+  }
 
   if (adapter.translated) {
     // Set for this process so `childEnv` carries it to the harness. assert-ai
@@ -1897,12 +2120,15 @@ async function runWithToken(
     say(`Finished ${done} of ${runs.length}.`);
     // The numbers before the link, because the link is where somebody goes when
     // the numbers are not enough -- and on this path they usually are.
-    await summarizeRuns(args, resolveApiKey(args), runs, { say });
+    const summaries = await summarizeRuns(args, resolveApiKey(args), runs, { say });
     if (surface) {
       say("");
       say(formatSurfaceLink(surface, { stream: process.stdout }));
     }
-    return { ok: done > 0, jobsRun: done, expected: runs.length };
+    // Carried out so a caller can gate on the pass rate. Returning only
+    // `ok: done > 0` says the runs executed, which is a different question from
+    // whether the agent behaved -- and a CI step needs the second one.
+    return { ok: done > 0, jobsRun: done, expected: runs.length, summaries };
   } finally {
     // Always, including on the throw paths above. The adapter holds a listening
     // socket and a set of live conversations with the agent; leaving it up after
@@ -1955,8 +2181,17 @@ export async function evalCommand(
     resolveAgent = null,
   } = {}
 ) {
-  const sub = commandArgs.find((value) => !value.startsWith("-")) || "run";
+  // Bare `preman eval` used to mean "volunteer this machine as a worker and
+  // wait for queued work", which is the opposite of what someone typing the
+  // name of the feature is asking for. It now means "measure my agent", and the
+  // worker role keeps its explicit spellings: `eval run`, `eval run --token`,
+  // and the `runner work` alias that says what it is.
+  //
+  // `--token` still implies the worker path with no subcommand, because a
+  // redeemed token is by definition work somebody else chose.
   const args = makeArgs(commandArgs);
+  const named = commandArgs.find((value) => !value.startsWith("-")) || "";
+  const sub = named || (String(args.value("--token", "") || "").trim() ? "run" : "");
 
   if (sub === "target") {
     // Its own command so the model call and the file write happen once, on
@@ -1980,6 +2215,25 @@ export async function evalCommand(
 
   if (sub === "doctor") {
     const projectPath = path.resolve(args.value("--path", process.cwd()));
+
+    // Two questions, because they have two different answers and people hit
+    // both. "Can this box run a harness" is local and is what the rest of this
+    // block measures. "Would a run started from the dashboard reach a machine"
+    // is the account's, lives on the server, and is invisible from here -- a
+    // laptop with a perfect local setup still shows `disconnected` if its
+    // daemon is not running, and that is exactly the confusion worth printing.
+    try {
+      const { accountReadiness, requireKey } = await import("./eval_start.js");
+      const state = await accountReadiness(args, requireKey(args));
+      process.stdout.write(`Account: ${state.state}\n  ${state.note}\n`);
+      if (state.next_step) process.stdout.write(`  ${state.next_step}\n`);
+      process.stdout.write("\n");
+    } catch (error) {
+      // A doctor that dies on the first failed question is not a doctor. The
+      // local half below is the half somebody ran this for.
+      process.stdout.write(`Account: could not be read (${error.message})\n\n`);
+    }
+
     let python;
     try {
       // Doctor answers "could a run happen here", so it has to resolve the same
@@ -2005,6 +2259,30 @@ export async function evalCommand(
     return { ok: true, python };
   }
 
+  // The user's door. With a selection or a target this is a direct run; with
+  // nothing it is a conversation, because "evaluate my agent" is not a complete
+  // request and the model is much better than a flag at turning it into one --
+  // it reads what the agent does and narrows to two or three behaviours, where
+  // a bare command would price every behaviour that happens to be switched on.
+  if (sub === "") {
+    const direct =
+      args.has("--only") ||
+      args.has("--yes") ||
+      args.has("--endpoint") ||
+      args.has("--target") ||
+      args.has("--preman-agent");
+    if (!direct) {
+      const { agentCommand } = await import("./agent.js");
+      return agentCommand(["Evaluate my agent."]);
+    }
+    return runForThisAccount(args, {
+      runnerLoop,
+      saveRunnerState,
+      readRunnerState,
+      deviceId,
+    });
+  }
+
   if (sub !== "run") throw new EvalError(`unknown eval subcommand: ${sub}${EVAL_HELP}`);
 
   // The token path is a different command wearing the same name, and the

package/bin/eval_entry.js

@@ -52,7 +52,10 @@ export const ENTRY_FILE = `${ENTRY_MODULE}.py`;
 export const BRIDGE_FILE = "agent_bridge.mjs";
 
 const SEARCH_TIMEOUT_MS = 120_000;
-const PROBE_TIMEOUT_MS = 60_000;
+// One real turn, after importing the customer's agent. GPUScraper alone takes
+// 37s to import from /mnt/c (2,227 modules), so 60s failed on the import plus
+// one model call. PREMAN_PROBE_TIMEOUT_S overrides for slower repos.
+const PROBE_TIMEOUT_MS = (Number(process.env.PREMAN_PROBE_TIMEOUT_S) || 180) * 1000;
 const MAX_TREE_PATHS = 4_000;
 const MAX_TREE_BYTES = 120_000;
 const MAX_SHORTLIST = 12;
@@ -392,7 +395,11 @@ export async function validateShim(file, { projectPath, log }) {
     cwd: projectPath,
   });
   if (ran.error?.code === "ETIMEDOUT") {
-    return { ok: false, why: "it did not answer within 60 seconds" };
+    return {
+      ok: false,
+      why: `it did not answer within ${PROBE_TIMEOUT_MS / 1000} seconds (importing the agent ` +
+        `counts; set PREMAN_PROBE_TIMEOUT_S to allow longer)`,
+    };
   }
   const line = String(ran.stdout || "")
     .split("\n")

package/bin/eval_generation_contract.py

@@ -0,0 +1,72 @@
+"""Stable identity of inputs that decide an eval suite's generated cases.
+
+The API and the packaged device harness use byte-identical copies. This module
+has no ASSERT or application imports so both environments compute the same hash.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any, Mapping
+
+
+def generation_spec_sha256(
+    spec: Mapping[str, Any], *, config_dir: Path | None = None
+) -> str | None:
+    """Hash generation inputs, excluding the run and measured target.
+
+    A curated taxonomy is shipped to the device at a different path, so its
+    contents enter the identity and its absolute path does not. Missing files
+    yield no identity: a caller should skip bank reuse in that case.
+    """
+    if not isinstance(spec, Mapping):
+        return None
+    pipeline = spec.get("pipeline") or {}
+    if not isinstance(pipeline, dict):
+        return None
+    test_set = pipeline.get("test_set") or {}
+    if not isinstance(test_set, dict):
+        return None
+    test_set = dict(test_set)
+    taxonomy_path = test_set.pop("taxonomy_path", None)
+    taxonomy_hash = None
+    if taxonomy_path:
+        try:
+            path = Path(taxonomy_path)
+            if not path.is_absolute():
+                if config_dir is None:
+                    return None
+                path = Path(config_dir) / path
+            taxonomy_hash = hashlib.sha256(path.read_bytes()).hexdigest()
+        except (OSError, TypeError, ValueError):
+            return None
+
+    inference = pipeline.get("inference") or {}
+    target = inference.get("target") if isinstance(inference, dict) else {}
+    if not isinstance(target, dict):
+        target = {}
+    target_inputs = {"system_prompt": target.get("system_prompt")}
+    if test_set.get("tool_source") == "per_test_case":
+        target_inputs.update({
+            "model": target.get("model"),
+            "tools": target.get("tools"),
+        })
+
+    inputs = {
+        "format": "preman.eval-generation-input/v1",
+        "behavior": spec.get("behavior"),
+        "context": spec.get("context"),
+        "default_model": spec.get("default_model"),
+        "dimensions": spec.get("dimensions"),
+        "models": spec.get("models"),
+        "systematize": pipeline.get("systematize"),
+        "test_set": test_set,
+        "target_generation_inputs": target_inputs,
+        "taxonomy_sha256": taxonomy_hash,
+    }
+    try:
+        encoded = json.dumps(inputs, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    except (TypeError, ValueError):
+        return None
+    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()