self-bench 0.3.0 → 0.3.3

package/src/audit.ts

@@ -0,0 +1,106 @@
+import type { Difficulty, TaskDefinition } from "./contracts.js";
+
+export interface StaticAuditReport {
+  readonly accepted: boolean;
+  readonly blockers: readonly string[];
+  readonly metrics: {
+    readonly implementationFiles: number;
+    readonly implementationChangedLines: number;
+    readonly testFiles: number;
+  };
+}
+
+const thresholds: Record<
+  Difficulty,
+  {
+    readonly implementationFiles: number;
+    readonly changedLines: number;
+    readonly passToPass: number;
+  }
+> = {
+  easy: { implementationFiles: 1, changedLines: 20, passToPass: 0 },
+  medium: { implementationFiles: 2, changedLines: 50, passToPass: 1 },
+  hard: { implementationFiles: 3, changedLines: 100, passToPass: 2 },
+};
+
+export function auditTaskDefinition(
+  definition: TaskDefinition,
+  goldPatch: string,
+  testPatch: string,
+): StaticAuditReport {
+  const gold = patchMetrics(goldPatch);
+  const tests = patchMetrics(testPatch);
+  const threshold = thresholds[definition.difficulty];
+  const testPathSet = new Set(tests.files);
+  const blockers: string[] = [];
+  const overlap = gold.files.filter((path) => testPathSet.has(path));
+  if (overlap.length > 0) {
+    blockers.push(`gold and held-out test patches overlap: ${overlap.join(", ")}`);
+  }
+  if (gold.files.length < threshold.implementationFiles) {
+    blockers.push(
+      `${definition.difficulty} mode requires at least ${threshold.implementationFiles} implementation files; found ${gold.files.length}`,
+    );
+  }
+  if (gold.changedLines < threshold.changedLines) {
+    blockers.push(
+      `${definition.difficulty} mode requires at least ${threshold.changedLines} changed implementation lines; found ${gold.changedLines}`,
+    );
+  }
+  if (tests.files.length === 0) {
+    blockers.push("held-out test patch changes no files");
+  }
+  if (definition.passToPass.length < threshold.passToPass) {
+    blockers.push(
+      `${definition.difficulty} mode requires at least ${threshold.passToPass} pass-to-pass regression tests`,
+    );
+  }
+  if (
+    definition.failToPass.some((path) => testCommandHardcodesPath(definition.testCommand, path)) ||
+    definition.passToPass.some((path) => testCommandHardcodesPath(definition.testCommand, path))
+  ) {
+    blockers.push(
+      'test command must not hard-code fail-to-pass or pass-to-pass paths outside "{tests}"',
+    );
+  }
+  if (definition.testCommand.split("{tests}").length !== 2) {
+    blockers.push('test command must contain "{tests}" exactly once');
+  }
+  if (/(["'])\{tests\}\1/.test(definition.testCommand)) {
+    blockers.push('test command must expand "{tests}" as an unquoted shell argument list');
+  }
+  return {
+    accepted: blockers.length === 0,
+    blockers,
+    metrics: {
+      implementationFiles: gold.files.length,
+      implementationChangedLines: gold.changedLines,
+      testFiles: tests.files.length,
+    },
+  };
+}
+
+function testCommandHardcodesPath(command: string, path: string): boolean {
+  return command.replace("{tests}", "").includes(path);
+}
+
+function patchMetrics(patch: string): { files: string[]; changedLines: number } {
+  const files: string[] = [];
+  let changedLines = 0;
+  for (const line of patch.split("\n")) {
+    if (line.startsWith("diff --git a/")) {
+      const match = /^diff --git a\/(.+) b\/(.+)$/.exec(line);
+      if (match?.[2]) {
+        files.push(match[2]);
+      }
+      continue;
+    }
+    if (
+      (line.startsWith("+") && !line.startsWith("+++")) ||
+      (line.startsWith("-") && !line.startsWith("---"))
+    ) {
+      changedLines += 1;
+    }
+  }
+  return { files, changedLines };
+}

package/src/build-metadata.ts

@@ -0,0 +1,3 @@
+const unsetBuildCommit = "0".repeat(40);
+
+export const buildCommit = process.env.SELFBENCH_BUILD_COMMIT ?? unsetBuildCommit;

package/src/cli.ts

@@ -0,0 +1,359 @@
+#!/usr/bin/env node
+
+import { createHash } from "node:crypto";
+import { open, rm } from "node:fs/promises";
+import { homedir } from "node:os";
+import { dirname, resolve } from "node:path";
+import { Readable, Transform } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import type { ReadableStream as NodeReadableStream } from "node:stream/web";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+import { buildCommit } from "./build-metadata.js";
+import { runCommand } from "./process.js";
+import { collectGitHubPullRequestProvenance, collectRepositoryProvenance } from "./provenance.js";
+import { type PolledRunStatus, waitForRun } from "./run-wait.js";
+
+const [command, ...rest] = process.argv.slice(2);
+switch (command) {
+  case "up":
+    await up(rest);
+    break;
+  case "run":
+    await run(rest);
+    break;
+  case "down":
+    await down();
+    break;
+  case "status":
+    await passthrough("GET", `/v1/runs/${requiredArgument(rest, "run ID")}`);
+    break;
+  case "cancel":
+    await passthrough("POST", `/v1/runs/${requiredArgument(rest, "run ID")}/cancel`);
+    break;
+  case "list":
+    await passthrough("GET", "/v1/runs");
+    break;
+  case "download":
+    await download(requiredArgument(rest, "run ID"), rest[1] ?? fail("output path is required"));
+    break;
+  case "help":
+  case "--help":
+  case "-h":
+  case undefined:
+    printHelp();
+    break;
+  default:
+    throw new Error(`unknown command: ${command}`);
+}
+
+async function up(args: string[]): Promise<void> {
+  const parsed = parseArgs({
+    args,
+    options: {
+      backend: { type: "string", default: "docker" },
+      "modal-config": { type: "string" },
+    },
+    strict: true,
+  });
+  if (parsed.values.backend !== "docker" && parsed.values.backend !== "modal") {
+    fail('--backend must be "docker" or "modal"');
+  }
+  if (parsed.values.backend === "docker" && parsed.values["modal-config"] !== undefined) {
+    fail("--modal-config requires --backend modal");
+  }
+
+  const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+  const composeFile = resolve(projectRoot, "compose.yaml");
+  const backend = parsed.values.backend;
+  const environment = {
+    ...process.env,
+    SELFBENCH_BUILD_COMMIT: await resolveSelfBenchCommit(),
+    SELFBENCH_EXECUTION_BACKEND: backend,
+    SELFBENCH_HARBOR_ENVIRONMENT: backend,
+    ...(backend === "modal"
+      ? {
+          SELFBENCH_MODAL_CONFIG_PATH: resolve(
+            parsed.values["modal-config"] ?? resolve(homedir(), ".modal.toml"),
+          ),
+        }
+      : {}),
+  };
+
+  if (backend === "docker") {
+    await runCommand(
+      "docker",
+      [
+        "build",
+        "-f",
+        resolve(projectRoot, "Dockerfile.sandbox"),
+        "-t",
+        "selfbench-sandbox:local",
+        projectRoot,
+      ],
+      { env: environment },
+    );
+  }
+  await runCommand("docker", ["compose", "--file", composeFile, "up", "-d", "--build"], {
+    env: environment,
+  });
+  console.log(`SelfBench is running with the ${backend} backend at http://127.0.0.1:8080`);
+}
+
+async function down(): Promise<void> {
+  const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+  await runCommand("docker", ["compose", "--file", resolve(projectRoot, "compose.yaml"), "down"]);
+}
+
+async function run(args: string[]): Promise<void> {
+  const parsed = parseArgs({
+    args,
+    options: {
+      repo: { type: "string", short: "r" },
+      "easy-count": { type: "string" },
+      "medium-count": { type: "string" },
+      "hard-count": { type: "string" },
+      "run-id": { type: "string" },
+      model: { type: "string", default: "gpt-5.6-sol" },
+      wait: { type: "boolean", default: false },
+      output: { type: "string", short: "o" },
+    },
+    strict: true,
+  });
+  const repositoryPath = resolve(parsed.values.repo ?? fail("--repo is required"));
+  const candidateCounts = {
+    easy: nonnegativeInteger(parsed.values["easy-count"] ?? "0", "--easy-count"),
+    medium: nonnegativeInteger(parsed.values["medium-count"] ?? "0", "--medium-count"),
+    hard: nonnegativeInteger(parsed.values["hard-count"] ?? "0", "--hard-count"),
+  };
+  const totalCandidates = candidateCounts.easy + candidateCounts.medium + candidateCounts.hard;
+  if (totalCandidates < 1 || totalCandidates > 100) {
+    fail("the total candidate count must be between 1 and 100");
+  }
+  const runId = parsed.values["run-id"] ?? defaultRunId();
+  const [repository, localMessages, selfbenchCommit] = await Promise.all([
+    resolveRepository(repositoryPath),
+    collectRepositoryProvenance(repositoryPath, process.env.HOME ?? homedir()),
+    resolveSelfBenchCommit(),
+  ]);
+  const githubMessages = await collectGitHubPullRequestProvenance(repository.url);
+  const messages = [...localMessages, ...githubMessages];
+  if (messages.length === 0) {
+    throw new Error("no sanitized local-session or GitHub pull-request provenance was found");
+  }
+  const corpus = Buffer.from(`${messages.map((message) => JSON.stringify(message)).join("\n")}\n`);
+  const provenance = await requestJson(`/v1/provenance?runId=${encodeURIComponent(runId)}`, {
+    method: "POST",
+    body: corpus,
+    contentType: "application/x-ndjson",
+  });
+  const response = await requestJson("/v1/runs", {
+    method: "POST",
+    body: Buffer.from(
+      JSON.stringify({
+        runId,
+        repository,
+        provenance,
+        candidateCounts,
+        authoringModel: parsed.values.model,
+        selfbenchCommit,
+      }),
+    ),
+    contentType: "application/json",
+  });
+  console.log(
+    JSON.stringify(
+      {
+        ...response,
+        provenanceMessages: messages.length,
+        localProvenanceMessages: localMessages.length,
+        githubPullRequestMessages: githubMessages.length,
+      },
+      null,
+      2,
+    ),
+  );
+  if (!parsed.values.wait && parsed.values.output === undefined) {
+    return;
+  }
+  const status = await waitForRun({
+    poll: async () =>
+      asPolledRunStatus(
+        await requestJson(`/v1/runs/${encodeURIComponent(runId)}`, { method: "GET" }),
+      ),
+    onPhase: (current) => {
+      console.error(
+        JSON.stringify({
+          runId,
+          phase: current.phase,
+          accepted: current.accepted,
+          rejected: current.rejected,
+        }),
+      );
+    },
+  });
+  if (parsed.values.output !== undefined) {
+    await download(runId, parsed.values.output);
+  } else {
+    console.log(JSON.stringify(status, null, 2));
+  }
+}
+
+async function resolveRepository(path: string): Promise<{ url: string; commit: string }> {
+  const [remote, commit] = await Promise.all([
+    runCommand("git", ["-C", path, "remote", "get-url", "origin"]),
+    runCommand("git", ["-C", path, "rev-parse", "HEAD"]),
+  ]);
+  return { url: normalizeGitUrl(remote.stdout.trim()), commit: commit.stdout.trim() };
+}
+
+function normalizeGitUrl(value: string): string {
+  const ssh = /^git@github\.com:(.+)$/.exec(value);
+  if (ssh?.[1]) {
+    return `https://github.com/${ssh[1]}`;
+  }
+  if (value.startsWith("https://")) {
+    return value;
+  }
+  throw new Error(`unsupported origin URL: ${value}`);
+}
+
+async function resolveSelfBenchCommit(): Promise<string> {
+  if (process.env.SELFBENCH_BUILD_COMMIT) {
+    return process.env.SELFBENCH_BUILD_COMMIT;
+  }
+  if (/^[0-9a-f]{40}$/i.test(buildCommit) && !/^0+$/.test(buildCommit)) {
+    return buildCommit;
+  }
+  const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+  try {
+    return (await runCommand("git", ["-C", root, "rev-parse", "HEAD"])).stdout.trim();
+  } catch {
+    throw new Error(
+      "SelfBench was built without commit metadata; set SELFBENCH_BUILD_COMMIT to a full commit SHA",
+    );
+  }
+}
+
+async function passthrough(method: "GET" | "POST", path: string): Promise<void> {
+  console.log(JSON.stringify(await requestJson(path, { method }), null, 2));
+}
+
+async function download(runId: string, outputPath: string): Promise<void> {
+  const base = process.env.SELFBENCH_API_URL ?? "http://127.0.0.1:8080";
+  const headers = new Headers();
+  if (process.env.SELFBENCH_API_TOKEN) {
+    headers.set("authorization", `Bearer ${process.env.SELFBENCH_API_TOKEN}`);
+  }
+  const response = await fetch(new URL(`/v1/runs/${encodeURIComponent(runId)}/export`, base), {
+    headers,
+  });
+  if (!response.ok) {
+    const value = (await response.json()) as Record<string, unknown>;
+    throw new Error(String(value.error ?? `SelfBench API returned ${response.status}`));
+  }
+  const expectedSha256 = response.headers.get("x-content-sha256");
+  if (!expectedSha256 || !response.body) {
+    throw new Error("SelfBench API returned an export without integrity metadata");
+  }
+  const destination = resolve(outputPath);
+  const file = await open(destination, "wx");
+  let verified = false;
+  try {
+    const hash = createHash("sha256");
+    const hasher = new Transform({
+      transform(chunk: Buffer, _encoding, callback) {
+        hash.update(chunk);
+        callback(undefined, chunk);
+      },
+    });
+    await pipeline(
+      Readable.fromWeb(response.body as unknown as NodeReadableStream),
+      hasher,
+      file.createWriteStream(),
+    );
+    if (hash.digest("hex") !== expectedSha256) {
+      throw new Error("downloaded export failed its SHA-256 integrity check");
+    }
+    verified = true;
+  } finally {
+    await file.close().catch(() => undefined);
+    if (!verified) {
+      await rm(destination, { force: true });
+    }
+  }
+  console.log(JSON.stringify({ runId, output: destination }, null, 2));
+}
+
+async function requestJson(
+  path: string,
+  options: { method: "GET" | "POST"; body?: Uint8Array; contentType?: string },
+): Promise<Record<string, unknown>> {
+  const base = process.env.SELFBENCH_API_URL ?? "http://127.0.0.1:8080";
+  const headers = new Headers();
+  if (options.contentType) {
+    headers.set("content-type", options.contentType);
+  }
+  if (process.env.SELFBENCH_API_TOKEN) {
+    headers.set("authorization", `Bearer ${process.env.SELFBENCH_API_TOKEN}`);
+  }
+  const response = await fetch(new URL(path, base), {
+    method: options.method,
+    headers,
+    ...(options.body ? { body: Buffer.from(options.body) } : {}),
+  });
+  const value = (await response.json()) as Record<string, unknown>;
+  if (!response.ok) {
+    throw new Error(String(value.error ?? `SelfBench API returned ${response.status}`));
+  }
+  return value;
+}
+
+function requiredArgument(args: string[], label: string): string {
+  return args[0] ?? fail(`${label} is required`);
+}
+
+function asPolledRunStatus(value: Record<string, unknown>): PolledRunStatus {
+  if (typeof value.phase !== "string") {
+    throw new Error("SelfBench status response is missing its phase");
+  }
+  return { ...value, phase: value.phase };
+}
+
+function nonnegativeInteger(value: string, label: string): number {
+  const parsed = Number(value);
+  if (!Number.isInteger(parsed) || parsed < 0) {
+    throw new Error(`${label} must be a nonnegative integer`);
+  }
+  return parsed;
+}
+
+function defaultRunId(): string {
+  return `sb-${new Date().toISOString().replace(/\D/g, "").slice(0, 14)}-${crypto.randomUUID().slice(0, 8)}`;
+}
+
+function fail(message: string): never {
+  throw new Error(message);
+}
+
+function printHelp(): void {
+  console.log(`SelfBench creates durable tiered Harbor evaluations.
+
+Usage:
+  self-bench up [--backend docker|modal] [--modal-config PATH]
+  self-bench down
+  self-bench run --repo PATH [--easy-count N] [--medium-count N] [--hard-count N]
+                  [--model MODEL] [--run-id ID] [--wait] [--output OUTPUT.tar.gz]
+  self-bench status RUN_ID
+  self-bench cancel RUN_ID
+  self-bench download RUN_ID OUTPUT.tar.gz
+  self-bench list
+
+The up command starts the local stack and configures both sandbox execution and Harbor validation for
+one backend. Modal uses ~/.modal.toml unless --modal-config overrides it.
+
+The tier counts are candidate authoring budgets, not accepted-task targets. Rejected candidates are not
+replaced, and the export contains only accepted tasks. The run command performs only repository metadata
+and sanitized provenance upload locally; discovery, authoring, validation, review, and audit run remotely.
+--output implies --wait, blocks until completion, and downloads the SHA-256-verified export.`);
+}

package/src/codex-review.ts

@@ -0,0 +1,229 @@
+import { z } from "zod";
+import type { CouplingEvidence } from "./coupling.js";
+
+export const COUPLING_REVIEW_MODEL = "gpt-5.6-sol";
+
+export const couplingReviewSchema = z.object({
+  verdict: z.enum(["clean", "coupled"]),
+  reason: z.string().min(1),
+  findings: z
+    .array(
+      z.object({
+        artifact: z.string().min(1),
+        category: z.enum(["endpoint_path", "field_name", "header_name", "media_type", "other"]),
+        disposition: z.enum([
+          "base_contract",
+          "prompt_contract",
+          "external_contract",
+          "gold_only",
+          "not_contract",
+        ]),
+        evidence: z.string().min(1),
+      }),
+    )
+    .min(1),
+  counterexample: z.string().min(1),
+});
+
+export type CouplingReview = z.infer<typeof couplingReviewSchema>;
+
+export async function reviewCouplingWithCodex(input: {
+  readonly apiKey?: string;
+  readonly authJson?: string;
+  readonly prompt: string;
+  readonly signal?: AbortSignal;
+  readonly fetch?: typeof fetch;
+}): Promise<CouplingReview> {
+  const subscription =
+    !input.apiKey && input.authJson ? parseCredential(input.authJson) : undefined;
+  if (!input.apiKey && !subscription) {
+    throw new Error("OpenAI model authentication is required");
+  }
+  const request = input.fetch ?? fetch;
+  const response = await request(
+    subscription
+      ? "https://chatgpt.com/backend-api/codex/responses"
+      : "https://api.openai.com/v1/responses",
+    {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${subscription?.access ?? input.apiKey}`,
+        ...(subscription ? { "ChatGPT-Account-Id": accountIdFromToken(subscription.access) } : {}),
+        Originator: "selfbench",
+        "OpenAI-Beta": "responses=experimental",
+        Accept: "text/event-stream",
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        model: COUPLING_REVIEW_MODEL,
+        store: false,
+        stream: true,
+        instructions:
+          "You are an independent benchmark-quality reviewer. Analyze the supplied evidence and call submit_review exactly once.",
+        input: [
+          {
+            role: "user",
+            content: [{ type: "input_text", text: input.prompt }],
+          },
+        ],
+        reasoning: { effort: "high", summary: "auto" },
+        text: { verbosity: "low" },
+        include: ["reasoning.encrypted_content"],
+        tools: [reviewTool()],
+        tool_choice: { type: "function", name: "submit_review" },
+        parallel_tool_calls: false,
+      }),
+      ...(input.signal ? { signal: input.signal } : {}),
+    },
+  );
+  const responseText = await response.text();
+  if (!response.ok) {
+    throw new Error(
+      `Codex coupling review failed (${response.status}): ${responseText.slice(0, 500)}`,
+    );
+  }
+  return parseCodexReviewEvents(responseText);
+}
+
+function parseCredential(value: string): { access: string } {
+  const parsed = JSON.parse(value) as unknown;
+  const credential = isRecord(parsed) ? parsed["openai-codex"] : undefined;
+  if (!isRecord(credential) || typeof credential.access !== "string") {
+    throw new Error("Pi auth does not contain an OpenAI Codex subscription access token");
+  }
+  return { access: credential.access };
+}
+
+function accountIdFromToken(token: string): string {
+  const payload = JSON.parse(
+    Buffer.from(token.split(".")[1] ?? "", "base64url").toString("utf8"),
+  ) as unknown;
+  const auth = isRecord(payload) ? payload["https://api.openai.com/auth"] : undefined;
+  const accountId = isRecord(auth) ? auth.chatgpt_account_id : undefined;
+  if (typeof accountId !== "string" || !accountId) {
+    throw new Error("OpenAI Codex subscription token has no ChatGPT account ID");
+  }
+  return accountId;
+}
+
+export function parseCodexReviewEvents(value: string): CouplingReview {
+  for (const line of value.split("\n")) {
+    if (!line.startsWith("data: ")) {
+      continue;
+    }
+    const data = line.slice("data: ".length);
+    if (data === "[DONE]") {
+      continue;
+    }
+    const event = JSON.parse(data) as unknown;
+    if (!isRecord(event)) {
+      continue;
+    }
+    const item = event.item;
+    if (
+      event.type === "response.output_item.done" &&
+      isRecord(item) &&
+      item.type === "function_call" &&
+      item.name === "submit_review" &&
+      typeof item.arguments === "string"
+    ) {
+      return couplingReviewSchema.parse(JSON.parse(item.arguments));
+    }
+    if (event.type === "error") {
+      throw new Error(`Codex coupling review stream failed: ${JSON.stringify(event)}`);
+    }
+  }
+  throw new Error("Codex coupling review returned no submit_review tool call");
+}
+
+export function couplingReviewPrompt(): string {
+  return `Independently review this hard benchmark for test-to-gold coupling.
+
+For every exact endpoint path, response/request field, header, media type, helper/module, error string, schema/index name, UI copy/order, or other implementation artifact asserted by the held-out tests, cite either the authentic prompt text that requires it or deterministic evidence that the exact base repository already establishes it. A name merely appearing in the gold patch is not evidence. Reject a test that would fail a coherent implementation of the request using different names, file boundaries, payload presentation, or internal structure.
+
+Resolve every artifact listed in couplingEvidence.blockers with a finding whose artifact exactly matches it. Use external_contract only when the authentic request names a standard protocol that fixes the exact artifact independently of the gold patch, and cite that protocol. Use not_contract only for incidental test-language or framework syntax that is not an asserted product contract. A clean verdict with a missing blocker finding or a gold_only finding is rejected automatically.
+
+Your findings must identify the artifact, classify it, give its disposition, and cite concrete evidence. The counterexample must describe a plausible alternative correct implementation and whether the tests would accept it. Also reject prompt leakage of PRs, commits, test names, or the solution.`;
+}
+
+export function couplingReviewInput(
+  prompt: string,
+  testPatch: string,
+  goldPatch: string,
+  evidence: CouplingEvidence,
+): string {
+  return `${couplingReviewPrompt()}
+
+# Authentic request
+
+${prompt}
+
+# Deterministic base-contract evidence
+
+The exact-artifact scan compared the held-out tests with the authentic request, gold additions, and exact base snapshot. Every blocker below is asserted by tests and introduced by gold, but absent verbatim from both request and base. Resolve each blocker explicitly.
+
+${JSON.stringify(evidence, null, 2)}
+
+# Held-out test patch
+
+\`\`\`diff
+${testPatch}
+\`\`\`
+
+# Gold implementation patch
+
+\`\`\`diff
+${goldPatch}
+\`\`\`
+`;
+}
+
+function reviewTool(): Record<string, unknown> {
+  return {
+    type: "function",
+    name: "submit_review",
+    description: "Submit the final independent test-coupling verdict exactly once.",
+    strict: true,
+    parameters: {
+      type: "object",
+      additionalProperties: false,
+      required: ["verdict", "reason", "findings", "counterexample"],
+      properties: {
+        verdict: { type: "string", enum: ["clean", "coupled"] },
+        reason: { type: "string", minLength: 1 },
+        findings: {
+          type: "array",
+          minItems: 1,
+          items: {
+            type: "object",
+            additionalProperties: false,
+            required: ["artifact", "category", "disposition", "evidence"],
+            properties: {
+              artifact: { type: "string", minLength: 1 },
+              category: {
+                type: "string",
+                enum: ["endpoint_path", "field_name", "header_name", "media_type", "other"],
+              },
+              disposition: {
+                type: "string",
+                enum: [
+                  "base_contract",
+                  "prompt_contract",
+                  "external_contract",
+                  "gold_only",
+                  "not_contract",
+                ],
+              },
+              evidence: { type: "string", minLength: 1 },
+            },
+          },
+        },
+        counterexample: { type: "string", minLength: 1 },
+      },
+    },
+  };
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}