rovecode 0.4.0-beta.2 → 0.4.0

package/src/eval/record.ts

@@ -0,0 +1,202 @@
+/**
+ * Evaluate-and-record: runs one gauntlet task through the scripted (offline) runner,
+ * captures every loop event into a versioned trajectory JSONL (eval P0-1), then grades
+ * the end state with patch/test-based graders (eval P0-2) and writes the result line.
+ *
+ * The existing deterministic gauntlet is NOT modified: runTask gains only an optional
+ * event sink (same loop, same scripted stream, same verdict), and the grader layer is
+ * additive — a recorded run can be graded without touching gauntlet.ts's own verify.
+ */
+
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { runTask } from "./gauntlet-runner.ts";
+import type { GauntletTask, GauntletTranscript } from "./gauntlet.ts";
+import { runGraders, gradersPassed, type GraderOutcome, type GraderSpec } from "./grader.ts";
+import {
+  TrajectoryWriter,
+  snapshotFixture,
+  gitCommit,
+  makeRunId,
+  sha256Text,
+  portablize,
+  redactSafe,
+  TRAJECTORY_SCHEMA_VERSION,
+  type FailureTaxonomy,
+  type FixtureSpec,
+  type TrajectoryResultLine,
+  type TrajectoryStepLine,
+  type RecordedToolResult,
+} from "./trajectory.ts";
+import type { RunEvent, PermissionRule } from "../core/types.ts";
+
+export interface EvalTaskSpec {
+  id: string;
+  category: string;
+  prompt: string;
+  task: GauntletTask;
+  graderSpecs: GraderSpec[];
+  model?: { provider: string; model: string } | null;
+  seed?: number;
+  commit?: string | null;
+  permissionRules?: PermissionRule[];
+}
+
+export interface RecordOptions {
+  /** output dir for the trajectory JSONL (created if missing) */
+  dir: string;
+  runId?: string;
+  guard?: Parameters<typeof runTask>[2];
+}
+
+export interface RecordOutcome {
+  runId: string;
+  path: string;
+  result: TrajectoryResultLine;
+  transcript: GauntletTranscript;
+  graderOutcomes: GraderOutcome[];
+  fixture: FixtureSpec;
+}
+
+const EVIDENCE_TOOLCALLS = 64;
+
+/** Map a loop event onto trajectory step lines. `seq` orders the file; tool outputs get
+ *  their replay-comparable sha here (normalized RAW output, redaction is lossy). */
+class StepBuilder {
+  private seq = 0;
+  private turn = 0;
+
+  from(ev: RunEvent, workspace: string): TrajectoryStepLine[] {
+    switch (ev.type) {
+      case "run_start":
+        this.turn = 0;
+        return [this.step({ type: "user", text: ev.goal })];
+      case "turn_start":
+        this.turn = ev.turn;
+        return [];
+      case "tool_execution_start":
+        return [this.step({ type: "assistant", turn: this.turn, toolCalls: [{ id: ev.callId, tool: ev.tool, args: ev.args }] })];
+      case "tool_execution_end": {
+        const normalized = portablize(ev.output, workspace) as unknown as string;
+        const rec: RecordedToolResult = {
+          callId: ev.callId,
+          ok: ev.ok,
+          output: normalized,
+          outputSha: sha256Text(normalized),
+        };
+        return [this.step({ type: "tool", turn: this.turn, results: [rec] })];
+      }
+      case "tool_call_failed": {
+        const output = `${ev.reason}: ${ev.detail}`;
+        const rec: RecordedToolResult = { callId: ev.callId, ok: false, output, outputSha: sha256Text(portablize(output, workspace) as unknown as string) };
+        return [this.step({ type: "tool", turn: this.turn, results: [rec] })];
+      }
+      case "turn_end":
+        return [this.step({ type: "assistant", turn: ev.turn, stopReason: ev.stopReason })];
+      case "run_end":
+        return [this.step({ type: "assistant", stopReason: ev.status, text: ev.summary })];
+      default:
+        return [];
+    }
+  }
+
+  private step(fields: Record<string, unknown>): TrajectoryStepLine {
+    return { kind: "step", seq: this.seq++, ...fields } as TrajectoryStepLine;
+  }
+}
+
+export function classifyFailure(input: {
+  graderOutcomes?: readonly GraderOutcome[];
+  events?: readonly RunEvent[];
+  error?: unknown;
+  transcript?: Pick<GauntletTranscript, "finalText" | "events" | "toolCalls" | "recovered">;
+}): { taxonomy: FailureTaxonomy; detail: string } {
+  if (input.error !== undefined) {
+    const msg = input.error instanceof Error ? input.error.message : String(input.error);
+    return { taxonomy: /timeout/i.test(msg) ? "timeout" : "provider-error", detail: msg };
+  }
+  const failed = (input.graderOutcomes ?? []).find((o) => !o.advisory && !o.pass);
+  if (failed) return { taxonomy: "verify-failed", detail: `${failed.name}: ${failed.detail.slice(0, 300)}` };
+  const events = input.events ?? [];
+  for (const ev of events) {
+    if (ev.type === "tool_call_failed" && ev.reason === "invalid_args") return { taxonomy: "invalid-args", detail: ev.detail.slice(0, 300) };
+    if (ev.type === "tool_call_failed" && ev.reason === "permission_denied") return { taxonomy: "permission-denied", detail: ev.detail.slice(0, 300) };
+  }
+  const guardBlocked = events.some((ev) => ev.type === "tool_execution_end" && ev.output.includes("loop guard: blocked"));
+  if (guardBlocked) return { taxonomy: "loop-guard", detail: "the tool guard stubbed repeated identical calls until the run ended" };
+  if (input.transcript && /error:/i.test(input.transcript.finalText)) {
+    return { taxonomy: "provider-error", detail: input.transcript.finalText.slice(0, 300) };
+  }
+  return { taxonomy: "unknown", detail: "run failed without a recognized signature" };
+}
+
+/** Run one task offline (scripted provider), record its trajectory, grade its end state. */
+export async function evaluateAndRecord(spec: EvalTaskSpec, opts: RecordOptions): Promise<RecordOutcome> {
+  const runId = opts.runId ?? makeRunId(spec.id);
+  const ws = spec.task.setup ? spec.task.setup() : mkdtempSync(join(tmpdir(), "rovecode-eval-"));
+  const t0 = Date.now();
+  const builder = new StepBuilder();
+  const events: RunEvent[] = [];
+  let transcript: GauntletTranscript | null = null;
+  let error: unknown;
+  try {
+    const fixture = snapshotFixture(ws);
+    const writer = new TrajectoryWriter(opts.dir, runId, ws);
+    writer.writeHeader({
+      kind: "header",
+      schemaVersion: TRAJECTORY_SCHEMA_VERSION,
+      runId,
+      startedAt: t0,
+      task: { id: spec.id, category: spec.category, prompt: spec.prompt },
+      model: spec.model ?? null,
+      seed: spec.seed ?? 0,
+      commit: spec.commit !== undefined ? spec.commit : gitCommit(),
+      fixture,
+      ...(spec.permissionRules ? { permissionRules: spec.permissionRules } : {}),
+      env: { platform: process.platform, runtime: "bun" },
+    });
+    const onEvent = (ev: RunEvent): void => {
+      events.push(ev);
+      for (const step of builder.from(ev, ws)) writer.writeStep(step);
+    };
+    try {
+      transcript = await runTask(spec.task, ws, opts.guard ?? undefined, onEvent);
+    } catch (e) {
+      error = e;
+    }
+    const graderOutcomes = error === undefined
+      ? await runGraders(spec.graderSpecs, { workspace: ws, fixture, transcript: transcript ?? { toolCalls: [], events: [], finalText: "", recovered: false } })
+      : [];
+    const pass = error === undefined && transcript !== null && gradersPassed(graderOutcomes);
+    const failure = pass ? undefined : classifyFailure({ graderOutcomes, events, error, transcript: transcript ?? undefined });
+    const result: TrajectoryResultLine = {
+      kind: "result",
+      schemaVersion: TRAJECTORY_SCHEMA_VERSION,
+      runId,
+      outcome: pass ? "pass" : failure?.taxonomy === "timeout" || failure?.taxonomy === "provider-error" ? "error" : "fail",
+      durationMs: Date.now() - t0,
+      toolCallCount: transcript?.toolCalls.length ?? 0,
+      ...(transcript?.usage ? { usage: transcript.usage } : {}),
+      graderSpecs: spec.graderSpecs as unknown[],
+      graderResults: graderOutcomes.map((o) => ({ name: o.name, pass: o.pass, detail: redactSafe(o.detail).slice(0, 400) })),
+      evidence: {
+        finalText: transcript?.finalText,
+        ...(transcript ? { toolCalls: transcript.toolCalls.slice(0, EVIDENCE_TOOLCALLS).map((c) => ({ tool: c.tool, args: c.args })) } : {}),
+        ...(failure ? { notes: [`${failure.taxonomy}: ${failure.detail}`] } : {}),
+      },
+      ...(failure ? { failure } : {}),
+    };
+    writer.writeResult(result);
+    return {
+      runId,
+      path: writer.path,
+      result,
+      transcript: transcript ?? { toolCalls: [], events: [], finalText: "", recovered: false },
+      graderOutcomes,
+      fixture,
+    };
+  } finally {
+    rmSync(ws, { recursive: true, force: true });
+  }
+}

package/src/eval/redact.ts

@@ -0,0 +1,141 @@
+/**
+ * Secrets redaction for eval persistence (eval P0-5).
+ *
+ * Every string that lands in a trajectory JSONL — tool args, tool outputs, fixture
+ * contents, evidence — goes through redactSecrets/redactDeep FIRST. The patterns cover
+ * the credential shapes the harness actually handles (provider keys, Bearer/Basic auth
+ * headers, JWTs, PEM private keys) plus the generic `key = "value"` assignments that
+ * leak keys through tool output. Sensitive KEY names in structured args are redacted
+ * regardless of value shape, because a value like `Bearer whatever` is not always
+ * pattern-visible.
+ *
+ * Scope note (the research critique's premise, verified by
+ * test/unit/eval-redact.test.ts): src/telemetry/otel.ts records ids, sizes and
+ * outcomes only — it never carries args or output, so it needs no redaction pass and
+ * this module deliberately does NOT touch it. If that contract ever erodes, the test
+ * pins it red.
+ *
+ * Determinism matters more than recall here: replays compare hashes of redacted-
+ * normalized text, so the replacement is a stable literal ("[REDACTED:<kind>]") and
+ * redaction is idempotent — a second pass changes nothing.
+ */
+
+export interface RedactionPattern {
+  name: string;
+  re: RegExp;
+  /** replacement for the whole match; $1… refer to capture groups */
+  replace?: string;
+}
+
+/** Ordered: a pattern consumes its match before later ones see it, so the more
+ *  specific provider shapes come first (sk-ant-… would otherwise be eaten by the
+ *  generic openai pattern), and the generic assignment pass runs last over text
+ *  that may already carry [REDACTED:…] tokens (still idempotent). */
+export const REDACTION_PATTERNS: readonly RedactionPattern[] = [
+  { name: "anthropic", re: /sk-ant-[A-Za-z0-9_-]{16,}/g },
+  { name: "openai", re: /sk-(?:proj-)?[A-Za-z0-9_-]{20,}/g },
+  { name: "aws-access-key", re: /AKIA[0-9A-Z]{16}/g },
+  { name: "github", re: /(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{22,})/g },
+  { name: "google", re: /AIza[0-9A-Za-z_-]{35,}/g },
+  { name: "slack", re: /xox[bpars]-[A-Za-z0-9-]{10,}/g },
+  { name: "private-key", re: /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY(?: BLOCK)?-----[\s\S]*?-----END (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY(?: BLOCK)?-----/g },
+  { name: "jwt", re: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{5,}\b/g },
+  { name: "bearer", re: /\b(Bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, replace: "$1 [REDACTED:bearer]" },
+  { name: "basic-auth", re: /\b(Basic)\s+[A-Za-z0-9+/=]{8,}/gi, replace: "$1 [REDACTED:basic-auth]" },
+];
+
+/** The literal a redacted value becomes. Stable across runs so replay hashes match. */
+export function redactionToken(kind: string): string {
+  return `[REDACTED:${kind}]`;
+}
+
+/** `key = "value"` / `password: value` — keep the key, lose the value. Done in code (not one
+ *  regex) because quoted and bare values end differently, and String.replace cannot consume
+ *  text beyond the match; an unterminated quote is left alone (redaction must not corrupt
+ *  non-secret text it cannot bound). */
+const ASSIGNMENT_RE = /\b(api[_-]?key|apikey|secret|access[_-]?token|auth[_-]?token|refresh[_-]?token|token|pass(?:word)?|passwd|pwd|client[_-]?secret)\b(\s*[:=]\s*)/gi;
+
+function redactAssignments(text: string, hits: Map<string, number>): string {
+  let out = "";
+  let last = 0;
+  for (const m of text.matchAll(ASSIGNMENT_RE)) {
+    const start = m.index ?? 0;
+    const valueStart = start + m[0].length;
+    const q = text[valueStart];
+    let end = valueStart;
+    let replacement: string;
+    if (q === '"' || q === "'") {
+      const close = text.indexOf(q, valueStart + 1);
+      if (close === -1) continue; // unterminated: leave this one untouched
+      end = close + 1;
+      replacement = `${m[1]}${m[2]}${q}${redactionToken("assignment")}${q}`;
+    } else {
+      let j = valueStart;
+      while (j < text.length && !/[\s,;)\]}]/.test(text[j]!)) j++;
+      if (j === valueStart) continue; // separator with no value — nothing to hide
+      end = j;
+      replacement = `${m[1]}${m[2]}${redactionToken("assignment")}`;
+    }
+    out += text.slice(last, start) + replacement;
+    last = end;
+    hits.set("assignment", (hits.get("assignment") ?? 0) + 1);
+  }
+  return out + text.slice(last);
+}
+
+function applyPatterns(text: string, hits: Map<string, number>): string {
+  let out = text;
+  for (const p of REDACTION_PATTERNS) {
+    const replacement = p.replace;
+    out = out.replace(p.re, (...args: unknown[]) => {
+      hits.set(p.name, (hits.get(p.name) ?? 0) + 1);
+      if (replacement === undefined) return redactionToken(p.name);
+      // expand $1…$9 against the callback's capture groups (args[0] is the match)
+      return replacement.replace(/\$(\d)/g, (_, d: string) => {
+        const g = args[Number(d)];
+        return typeof g === "string" ? g : "";
+      });
+    });
+  }
+  return redactAssignments(out, hits);
+}
+
+/** Report form: the redacted text plus one count per pattern that hit. */
+export function redactionReport(text: string): { text: string; matches: { kind: string; count: number }[] } {
+  const hits = new Map<string, number>();
+  const out = applyPatterns(text, hits);
+  return { text: out, matches: [...hits].map(([kind, count]) => ({ kind, count })) };
+}
+
+/** Plain-text form: just the redacted text. */
+export function redactSecrets(text: string): string {
+  return redactionReport(text).text;
+}
+
+const SENSITIVE_KEY = /^(?:.*[-_])?(?:pass(?:word)?|passwd|pwd|secret|token|api[-_]?key|apikey|authorization|private[-_]?key|credentials?)$/i;
+
+/** Deep form for structured payloads (tool args, fixture objects): every string value is
+ *  pattern-redacted, and values under sensitive KEY names are replaced wholesale — a
+ *  credential is still a credential when no pattern can name its shape. Cycles degrade to
+ *  a marker instead of hanging; non-string leaves pass through untouched. */
+export function redactDeep<T>(value: T, seen: Set<object> = new Set()): T {
+  if (typeof value === "string") return redactSecrets(value) as unknown as T;
+  if (value === null || typeof value !== "object") return value;
+  const obj = value as object;
+  if (seen.has(obj)) return "[circular]" as unknown as T;
+  seen.add(obj);
+  try {
+    if (Array.isArray(obj)) return obj.map((v) => redactDeep(v, seen)) as unknown as T;
+    const out: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(obj as Record<string, unknown>)) {
+      if (typeof v === "string" && SENSITIVE_KEY.test(k)) {
+        out[k] = redactionToken("key");
+      } else {
+        out[k] = redactDeep(v, seen);
+      }
+    }
+    return out as unknown as T;
+  } finally {
+    seen.delete(obj);
+  }
+}

package/src/eval/replay.ts

@@ -0,0 +1,147 @@
+/**
+ * Trajectory replay (eval P0-3): verify a recorded run WITHOUT a network or a model.
+ *
+ * The recorded trajectory carries a fixture snapshot (the pre-run workspace) and every
+ * tool call the model made. Replay rebuilds the workspace from the snapshot, re-executes
+ * the recorded calls through the REAL tool registry with the recorded permission rules,
+ * and compares, per call, the ok flag and the sha256 of the path-normalized RAW output
+ * against the recorded hashes. Then it re-runs the recorded grader specs and compares the
+ * composite verdict with the recorded outcome. Anything that drifted — a tampered call,
+ * a changed tool implementation, a nondeterministic command — is reported per step.
+ *
+ * Determinism guarantees this leans on: fixtures are files-on-disk, "<workspace>" tokens
+ * make recorded paths machine-independent (portablize/deportablize), and grader specs
+ * are workspace-relative. A tool whose output genuinely varies between machines (clocks,
+ * line endings in shell output) will honestly fail replay — that is the point.
+ */
+
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { readTrajectory, materializeFixture, portablize, sha256Text, deportablize } from "./trajectory.ts";
+import { runGraders, gradersPassed, GraderConfigError, type GraderOutcome, type GraderSpec } from "./grader.ts";
+import { ToolRegistry } from "../core/tools.ts";
+import type { PermissionRule, RunEvent, ToolCallPart } from "../core/types.ts";
+import { readTool, editTool, writeTool, bashTool } from "../coding/hashline.ts";
+import { globTool, grepTool, lsTool } from "../coding/files.ts";
+
+export const REPLAY_ALLOW_ALL: PermissionRule[] = [{ action: "*", resource: "*", effect: "allow" }];
+
+export interface StepComparison {
+  callId: string;
+  tool: string;
+  okMatch: boolean | null;       // null = no recorded result to compare against
+  hashMatch: boolean | null;     // null = recorded result had no hash
+  detail?: string;
+}
+
+export interface ReplayReport {
+  ok: boolean;
+  runId: string;
+  task: { id: string; category: string };
+  comparisons: StepComparison[];
+  graderOutcomes: GraderOutcome[];
+  /** replay's grader verdict vs the recorded outcome (null = nothing to compare) */
+  outcomeMatch: boolean | null;
+  mismatches: string[];
+}
+
+export interface ReplayOptions {
+  /** extra/override tools — defaults to the standard coding set */
+  registerTools?: (registry: ToolRegistry) => void;
+  /** keep the rebuilt workspace dir for debugging (default: removed) */
+  keepWorkspace?: boolean;
+}
+
+export async function replayTrajectory(path: string, opts?: ReplayOptions): Promise<ReplayReport> {
+  const t = readTrajectory(path);
+  const ws = mkdtempSync(join(tmpdir(), "rovecode-replay-"));
+  try {
+    materializeFixture(t.header.fixture, ws);
+
+    const registry = new ToolRegistry();
+    registry.register(readTool, editTool, writeTool, bashTool, globTool, grepTool, lsTool);
+    opts?.registerTools?.(registry);
+
+    const rules = t.header.permissionRules ?? REPLAY_ALLOW_ALL;
+    const recordedResults = new Map<string, { ok: boolean; outputSha?: string; output?: string }>();
+    for (const step of t.steps) {
+      for (const r of step.results ?? []) recordedResults.set(r.callId, { ok: r.ok, outputSha: r.outputSha, output: r.output });
+    }
+
+    const comparisons: StepComparison[] = [];
+    const mismatches: string[] = [];
+    const emit = (_ev: RunEvent): void => { /* replay is not recorded */ };
+
+    for (const step of t.steps) {
+      for (const call of step.toolCalls ?? []) {
+        const args = deportablize(call.args, ws);
+        const recorded = recordedResults.get(call.id);
+        const comparison: StepComparison = { callId: call.id, tool: call.tool, okMatch: null, hashMatch: null };
+        if (!recorded) {
+          comparison.detail = "no recorded result for this call";
+          mismatches.push(`${call.tool}(${call.id}): no recorded result`);
+          comparisons.push(comparison);
+          continue;
+        }
+        const part: ToolCallPart = { kind: "tool_call", id: call.id, tool: call.tool, args };
+        let out: { ok: boolean; output: string };
+        try {
+          out = await registry.dispatch(
+            part,
+            { sessionId: `replay-${t.header.runId}`, cwd: ws, signal: new AbortController().signal, permissions: { effect: "allow" } },
+            undefined,
+            rules,
+            undefined,
+            emit,
+          );
+        } catch (e) {
+          out = { ok: false, output: `replay dispatch threw: ${e instanceof Error ? e.message : String(e)}` };
+        }
+        comparison.okMatch = out.ok === recorded.ok;
+        const replaySha = sha256Text(portablize(out.output, ws) as unknown as string);
+        comparison.hashMatch = recorded.outputSha !== undefined ? replaySha === recorded.outputSha : null;
+        if (recorded.output !== undefined && comparison.hashMatch === false) {
+          comparison.detail = `output drifted — recorded: ${recorded.output.slice(0, 120)} | replayed: ${(portablize(out.output, ws) as unknown as string).slice(0, 120)}`;
+        }
+        if (comparison.okMatch === false) mismatches.push(`${call.tool}(${call.id}): ok ${out.ok} vs recorded ${recorded.ok}`);
+        if (comparison.hashMatch === false) mismatches.push(`${call.tool}(${call.id}): output hash drifted`);
+        comparisons.push(comparison);
+      }
+    }
+
+    let graderOutcomes: GraderOutcome[] = [];
+    let outcomeMatch: boolean | null = null;
+    const specs = (t.result?.graderSpecs ?? []) as GraderSpec[];
+    if (specs.length > 0) {
+      try {
+        graderOutcomes = await runGraders(specs, {
+          workspace: ws,
+          fixture: t.header.fixture,
+          transcript: { finalText: t.result?.evidence.finalText ?? "", toolCalls: [], events: [], recovered: true },
+        });
+        if (t.result) outcomeMatch = gradersPassed(graderOutcomes) === (t.result.outcome === "pass");
+      } catch (e) {
+        if (e instanceof GraderConfigError) {
+          mismatches.push(`grader config: ${e.message}`);
+        } else {
+          mismatches.push(`grader run failed: ${e instanceof Error ? e.message : String(e)}`);
+        }
+      }
+    }
+    if (outcomeMatch === false) mismatches.push("replayed graders disagree with the recorded outcome");
+
+    const stepMismatch = comparisons.some((c) => c.okMatch === false || c.hashMatch === false);
+    return {
+      ok: !stepMismatch && mismatches.length === 0 && outcomeMatch !== false,
+      runId: t.header.runId,
+      task: { id: t.header.task.id, category: t.header.task.category },
+      comparisons,
+      graderOutcomes,
+      outcomeMatch,
+      mismatches,
+    };
+  } finally {
+    if (!opts?.keepWorkspace) rmSync(ws, { recursive: true, force: true });
+  }
+}