self-bench 0.3.6 → 0.3.8

package/drizzle/meta/_journal.json

@@ -0,0 +1,48 @@
+{
+  "version": "7",
+  "dialect": "postgresql",
+  "entries": [
+    {
+      "idx": 0,
+      "version": "7",
+      "when": 1788564991375,
+      "tag": "0000_glossy_johnny_blaze",
+      "breakpoints": true
+    },
+    {
+      "idx": 1,
+      "version": "7",
+      "when": 1788659606426,
+      "tag": "0001_evaluation_records",
+      "breakpoints": true
+    },
+    {
+      "idx": 2,
+      "version": "7",
+      "when": 1788716380068,
+      "tag": "0002_task_deletion",
+      "breakpoints": true
+    },
+    {
+      "idx": 3,
+      "version": "7",
+      "when": 1788717620218,
+      "tag": "0003_drop_repo_runs",
+      "breakpoints": true
+    },
+    {
+      "idx": 4,
+      "version": "7",
+      "when": 1788993718151,
+      "tag": "0004_preserve_batch_runs",
+      "breakpoints": true
+    },
+    {
+      "idx": 5,
+      "version": "7",
+      "when": 1789417155739,
+      "tag": "0005_api_keys",
+      "breakpoints": true
+    }
+  ]
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "self-bench",
-  "version": "0.3.6",
+  "version": "0.3.8",
   "description": "Turn completed repository changes into durable, private Harbor evaluations",
   "license": "MIT",
   "repository": {
@@ -13,31 +13,27 @@
   },
   "type": "module",
   "bin": {
-    "self-bench": "dist/cli.js",
-    "self-bench-agent-smoke": "dist/agent-smoke-main.js",
-    "self-bench-eval": "dist/eval-main.js",
-    "self-bench-reaudit": "dist/reaudit-main.js",
-    "self-bench-repair": "dist/repair-main.js",
-    "self-bench-validate": "dist/validate-main.js"
+    "self-bench": "dist/cli.js"
   },
   "files": [
     "dist",
-    "src",
-    "review",
+    "src/extensions",
+    "src/skills",
+    "drizzle",
     "docs",
     "compose.yaml",
     "Dockerfile",
     "Dockerfile.sandbox",
     "bun.lock",
-    "scripts",
-    "biome.json",
-    "tsconfig.json",
-    "tsconfig.build.json",
     ".dockerignore"
   ],
   "engines": {
     "node": ">=22"
   },
+  "overrides": {
+    "nanoid": "3.3.18",
+    "uuid": "11.1.1"
+  },
   "scripts": {
     "build": "bun run clean && bun run build:server && bun run build:review",
     "clean": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"",
@@ -49,9 +45,10 @@
     "prepublishOnly": "bun run validate",
     "verify:package": "bun scripts/verify-package.ts",
     "verify:package:docker": "bun scripts/verify-package.ts --docker-build",
-    "check": "biome check . && tsc --noEmit -p tsconfig.json && bun run typecheck:review",
+    "check": "biome check . && tsc --noEmit -p tsconfig.json && bun run typecheck:review && bun scripts/check-file-sizes.ts",
     "dev:api": "bun run build:review && tsx src/api-main.ts",
     "dev:review": "vite --config review/vite.config.ts",
+    "dev:site": "bun scripts/dev-site.ts",
     "dev:worker": "bun run build:server && tsx src/temporal/worker-main.ts",
     "format": "biome format --write .",
     "start:api": "node dist/api-main.js",
@@ -62,30 +59,47 @@
   },
   "dependencies": {
     "@clack/prompts": "1.7.0",
-    "@google-cloud/storage": "^7.17.0",
-    "@temporalio/activity": "^1.13.2",
-    "@temporalio/client": "^1.13.2",
-    "@temporalio/common": "^1.13.2",
-    "@temporalio/worker": "^1.13.2",
-    "@temporalio/workflow": "^1.13.2",
+    "@google-cloud/storage": "^7.22.0",
+    "@openai/codex": "0.153.4",
+    "@sinclair/typebox": "0.34.41",
+    "@temporalio/activity": "^1.23.0",
+    "@temporalio/client": "^1.23.0",
+    "@temporalio/common": "^1.23.0",
+    "@temporalio/worker": "^1.23.0",
+    "@temporalio/workflow": "^1.23.0",
     "@vercel/sandbox": "3.0.0",
+    "drizzle-orm": "^0.45.2",
+    "e2b": "2.46.0",
     "modal": "^0.9.0",
-    "zod": "^4.1.12"
+    "postgres": "3.4.7",
+    "zod": "^4.5.4",
+    "@radix-ui/react-slot": "1.3.3",
+    "@radix-ui/react-dropdown-menu": "2.1.24",
+    "class-variance-authority": "0.7.1",
+    "clsx": "2.1.1",
+    "tailwind-merge": "3.6.0",
+    "lucide-react": "1.43.0"
   },
   "devDependencies": {
-    "@biomejs/biome": "^2.2.6",
+    "@tailwindcss/vite": "4.3.3",
+    "tailwindcss": "4.3.3",
+    "@biomejs/biome": "^2.5.11",
     "@earendil-works/pi-coding-agent": "0.84.0",
+    "@electric-sql/pglite": "0.3.15",
+    "@mupt-ai/dari-pareto": "0.2.0",
     "@pierre/diffs": "1.2.12",
-    "@sinclair/typebox": "0.34.41",
     "@types/bun": "^1.3.14",
     "@types/node": "24.10.13",
     "@types/react": "19.2.14",
     "@types/react-dom": "19.2.3",
     "@vitejs/plugin-react": "4.7.0",
+    "drizzle-kit": "^0.31.10",
+    "happy-dom": "20.14.1",
     "react": "19.2.7",
     "react-dom": "19.2.7",
-    "tsx": "^4.20.6",
+    "react-router": "7.18.3",
+    "tsx": "^4.23.13",
     "typescript": "5.9.3",
-    "vite": "7.3.1"
+    "vite": "7.3.6"
   }
 }

package/src/extensions/authoring.ts

@@ -1,86 +1,97 @@
-import { mkdirSync, writeFileSync } from "node:fs";
+import { cpSync, mkdirSync } from "node:fs";
 import { join } from "node:path";
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { Type } from "@sinclair/typebox";
+import { loadTaskDeliverable } from "./shared/deliverable.js";
+import { VerifyClient, verifyOutcomeText } from "./shared/mailbox.js";
+import {
+  requiredEnvironment,
+  runStaticCheck,
+  stageSubmission,
+  staticCheckFailure,
+} from "./shared/static-check.js";
 
-const taskDefinition = Type.Object(
-  {
-    schemaVersion: Type.Literal(1),
-    difficulty: Type.Union([Type.Literal("easy"), Type.Literal("medium"), Type.Literal("hard")]),
-    taskId: Type.String({ pattern: "^[A-Za-z0-9][A-Za-z0-9._-]*$" }),
-    repo: Type.String({ minLength: 1 }),
-    baseCommit: Type.String({ pattern: "^[0-9a-fA-F]{40}$" }),
-    workdir: Type.String({ minLength: 1 }),
-    setupCommand: Type.String({ minLength: 1 }),
-    testCommand: Type.String({ minLength: 1 }),
-    failToPass: Type.Array(Type.String({ minLength: 1 }), { minItems: 1 }),
-    passToPass: Type.Array(Type.String({ minLength: 1 })),
-    testPaths: Type.Array(Type.String({ minLength: 1 }), { minItems: 1 }),
-    toolchains: Type.Array(
-      Type.Union([
-        Type.Literal("uv"),
-        Type.Literal("bun"),
-        Type.Literal("go"),
-        Type.Literal("node"),
-        Type.Literal("python"),
-        Type.Literal("rust"),
-      ]),
-      { minItems: 1 },
-    ),
-    sourcePr: Type.Integer({ minimum: 1 }),
-    sourceUrl: Type.String({ minLength: 1 }),
-    prompt: Type.String({ minLength: 1 }),
-    timeouts: Type.Object({
-      setupSeconds: Type.Integer({ minimum: 1 }),
-      agentSeconds: Type.Integer({ minimum: 1 }),
-      testsSeconds: Type.Integer({ minimum: 1 }),
-    }),
-    resources: Type.Object({
-      cpus: Type.Number({ exclusiveMinimum: 0 }),
-      memoryMb: Type.Integer({ minimum: 1 }),
-      storageMb: Type.Integer({ minimum: 1 }),
-    }),
-  },
-  { additionalProperties: false },
-);
+const noArguments = Type.Object({}, { additionalProperties: false });
+
+/** The clean base tree the patches must apply to: /work/repo at the definition's base commit. */
+function applyTarget(definition: Record<string, unknown>): { base: string } {
+  return { base: typeof definition.baseCommit === "string" ? definition.baseCommit : "HEAD" };
+}
 
 export default function authoringExtension(pi: ExtensionAPI): void {
+  const client = new VerifyClient();
+  const deliverable = (): string => process.env.SELFBENCH_DELIVERABLE ?? "/work/task";
+
   pi.registerTool({
-    name: "submit_task",
-    label: "Submit SelfBench task",
-    description: "Submit exactly one complete task at the assigned difficulty.",
-    parameters: Type.Object(
-      {
-        definition: taskDefinition,
-        testPatch: Type.String({ minLength: 1 }),
-        goldPatch: Type.String({ minLength: 1 }),
-      },
-      { additionalProperties: false },
-    ),
-    async execute(_toolCallId, input) {
-      const root = process.env.SELFBENCH_TASK_OUTPUT;
-      if (!root) {
-        throw new Error("SELFBENCH_TASK_OUTPUT is not configured");
-      }
-      const { definition, testPatch, goldPatch } = input;
-      if (!definition?.taskId || typeof testPatch !== "string" || typeof goldPatch !== "string") {
-        throw new Error("submit_task received an incomplete task");
+    name: "verify",
+    label: "Verify SelfBench task",
+    description:
+      "Takes no arguments. Reads the deliverable in /work/task (definition.json, instruction.md, test.patch, gold.patch), runs the static check, then the harness's real verification: trusted compile, audit, image build, smoke, nop, and oracle. Blocks until the report is back (can take up to an hour). Budget-limited per session; the result says how many calls remain.",
+    parameters: noArguments,
+    async execute() {
+      const loaded = loadTaskDeliverable(deliverable());
+      if ("isError" in loaded) {
+        return loaded;
       }
-      const directory = join(root, definition.taskId);
-      mkdirSync(directory, { recursive: false });
-      writeFileSync(
-        join(directory, "definition.json"),
-        `${JSON.stringify(definition, null, 2)}\n`,
-        {
-          flag: "wx",
-        },
+      const staging = stageSubmission(
+        loaded.definition,
+        loaded.testPatch,
+        loaded.goldPatch,
+        "selfbench-verify-",
       );
-      writeFileSync(join(directory, "test.patch"), testPatch, { flag: "wx" });
-      writeFileSync(join(directory, "gold.patch"), goldPatch, { flag: "wx" });
+      try {
+        const verdict = runStaticCheck(staging.directory, [], applyTarget(loaded.definition));
+        if (!verdict.ok) {
+          return staticCheckFailure(verdict, "task");
+        }
+      } finally {
+        staging.dispose();
+      }
+      const outcome = await client.verify("task", loaded);
       return {
-        content: [{ type: "text", text: `Submitted ${definition.taskId}.` }],
-        details: { taskId: definition.taskId },
+        content: [{ type: "text", text: verifyOutcomeText(outcome, "submit_task") }],
+        details: { kind: outcome.kind, remaining: client.remaining },
+        ...(outcome.kind === "exhausted" ? { isError: true } : {}),
       };
     },
   });
+
+  pi.registerTool({
+    name: "submit_task",
+    label: "Submit SelfBench task",
+    description:
+      "Takes no arguments. Reads the deliverable in /work/task, runs the static check, and records the task; a passing submission ends the session. Run verify first and submit only what verify reported green.",
+    parameters: noArguments,
+    async execute() {
+      const root = requiredEnvironment("SELFBENCH_TASK_OUTPUT");
+      const loaded = loadTaskDeliverable(deliverable());
+      if ("isError" in loaded) {
+        return loaded;
+      }
+      const taskId = String(loaded.definition.taskId ?? "");
+      const staging = stageSubmission(loaded.definition, loaded.testPatch, loaded.goldPatch);
+      try {
+        const verdict = runStaticCheck(staging.directory, [], applyTarget(loaded.definition));
+        if (!verdict.ok) {
+          return staticCheckFailure(verdict, "submission");
+        }
+        const directory = join(root, taskId);
+        mkdirSync(root, { recursive: true });
+        mkdirSync(directory, { recursive: false });
+        cpSync(staging.directory, directory, { recursive: true });
+        const verified = client.verifiedGreen(loaded);
+        return {
+          content: [
+            {
+              type: "text",
+              text: `Submitted ${taskId}; the static check passed and the Harbor tree is dry-rendered under ${verdict.renderedDirectory ?? "/work/rendered"}. ${verified ? "This deliverable matches your last green verify, so the worker reuses that report." : "This deliverable was not verified green in this session, so the worker verifies it now and a round is spent if it fails."} Stop here and wait.`,
+            },
+          ],
+          details: { taskId, verified },
+        };
+      } finally {
+        staging.dispose();
+      }
+    },
+  });
 }

package/src/extensions/shared/deliverable.ts

@@ -0,0 +1,153 @@
+import { spawnSync } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { FIX_FIELDS } from "./schemas.js";
+import { failure, type ToolFailure } from "./static-check.js";
+
+export interface TaskDeliverable {
+  readonly definition: Record<string, unknown>;
+  readonly testPatch: string;
+  readonly goldPatch: string;
+}
+
+export const TASK_DELIVERABLE_FILES = [
+  "definition.json",
+  "instruction.md",
+  "test.patch",
+  "gold.patch",
+] as const;
+
+/**
+ * Authoring deliverable: `<root>/definition.json` (schema 2 with the environment contract),
+ * `instruction.md`, `test.patch`, `gold.patch`. instruction.md is authoritative for the prompt:
+ * `definition.prompt` is derived from it and must match when present.
+ */
+export function loadTaskDeliverable(root: string): TaskDeliverable | ToolFailure {
+  const missing = TASK_DELIVERABLE_FILES.filter((name) => !existsSync(join(root, name)));
+  if (missing.length > 0) {
+    return deliverableFailure(
+      root,
+      missing.map((name) => `${name} is missing`),
+    );
+  }
+  let definition: Record<string, unknown>;
+  try {
+    const parsed = JSON.parse(readFileSync(join(root, "definition.json"), "utf8")) as unknown;
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+      throw new Error("must be a JSON object");
+    }
+    definition = parsed as Record<string, unknown>;
+  } catch (error) {
+    return deliverableFailure(root, [
+      `definition.json is not valid JSON: ${error instanceof Error ? error.message : String(error)}`,
+    ]);
+  }
+  const instruction = readFileSync(join(root, "instruction.md"), "utf8").trim();
+  const testPatch = readFileSync(join(root, "test.patch"), "utf8");
+  const goldPatch = readFileSync(join(root, "gold.patch"), "utf8");
+  const problems: string[] = [];
+  if (!instruction) {
+    problems.push("instruction.md is empty");
+  }
+  if (typeof definition.prompt === "string" && definition.prompt.trim() !== instruction) {
+    problems.push(
+      "definition.json prompt differs from instruction.md; remove prompt or make them identical",
+    );
+  }
+  if (!testPatch.trim()) {
+    problems.push("test.patch is empty");
+  }
+  if (!goldPatch.trim()) {
+    problems.push("gold.patch is empty");
+  }
+  if (problems.length > 0) {
+    return deliverableFailure(root, problems);
+  }
+  return { definition: { ...definition, prompt: instruction }, testPatch, goldPatch };
+}
+
+export interface FixDeliverable {
+  readonly definition: Record<string, unknown>;
+  readonly testPatch: string;
+  readonly goldPatch: string;
+  readonly testPatchSource: "file" | "working-tree";
+  readonly original: { definition: string; testPatch: string; goldPatch: string };
+}
+
+/**
+ * Verifier deliverable: `<fixRoot>/definition.json` (optional; only environment, testCommand,
+ * failToPass, passToPass, testPaths, timeouts, resources are read and merged onto the compiled
+ * task's definition) and `<fixRoot>/test.patch` (optional; regenerated with `git diff` from the
+ * repository working tree when absent).
+ */
+export function loadFixDeliverable(
+  fixRoot: string,
+  taskDirectory: string,
+  repository: string,
+): FixDeliverable | ToolFailure {
+  const originalDefinition = join(taskDirectory, "definition.json");
+  const definition = JSON.parse(readFileSync(originalDefinition, "utf8")) as Record<
+    string,
+    unknown
+  >;
+  const fixDefinitionPath = join(fixRoot, "definition.json");
+  if (existsSync(fixDefinitionPath)) {
+    let fix: Record<string, unknown>;
+    try {
+      fix = JSON.parse(readFileSync(fixDefinitionPath, "utf8")) as Record<string, unknown>;
+    } catch (error) {
+      return deliverableFailure(fixRoot, [
+        `definition.json is not valid JSON: ${error instanceof Error ? error.message : String(error)}`,
+      ]);
+    }
+    for (const field of FIX_FIELDS) {
+      if (fix[field] !== undefined) {
+        definition[field] = fix[field];
+      }
+    }
+  }
+  const testPatchPath = join(fixRoot, "test.patch");
+  const fromFile = existsSync(testPatchPath);
+  const testPatch = fromFile ? readFileSync(testPatchPath, "utf8") : workingTreePatch(repository);
+  if (!testPatch.trim()) {
+    return deliverableFailure(fixRoot, [
+      fromFile ? "test.patch is empty" : "test.patch is absent and /work/repo has no changes",
+    ]);
+  }
+  return {
+    definition,
+    testPatch,
+    goldPatch: readFileSync(join(taskDirectory, "solution/gold.patch"), "utf8"),
+    testPatchSource: fromFile ? "file" : "working-tree",
+    original: {
+      definition: originalDefinition,
+      testPatch: join(taskDirectory, "tests/test.patch"),
+      goldPatch: join(taskDirectory, "solution/gold.patch"),
+    },
+  };
+}
+
+export function workingTreePatch(repository: string): string {
+  for (const args of [
+    ["add", "-N", "--all"],
+    ["diff", "--binary", "HEAD"],
+  ]) {
+    const result = spawnSync("git", ["-C", repository, ...args], {
+      encoding: "utf8",
+      maxBuffer: 256 * 1024 * 1024,
+    });
+    if (result.status !== 0) {
+      throw new Error(`git ${args[0]} failed: ${result.stderr.slice(-2_000)}`);
+    }
+    if (args[0] === "diff") {
+      return result.stdout;
+    }
+  }
+  return "";
+}
+
+function deliverableFailure(root: string, problems: readonly string[]): ToolFailure {
+  return failure(
+    `The deliverable in ${root} is incomplete; nothing was recorded. Fix every item and call the tool again:\n${problems.map((problem) => `- [files] ${problem}`).join("\n")}`,
+  );
+}

package/src/extensions/shared/mailbox.ts

@@ -0,0 +1,128 @@
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+const DEFAULT_MAILBOX = "/work/mailbox";
+const DEFAULT_POLL_MS = 2_000;
+const DEFAULT_TIMEOUT_MS = 60 * 60 * 1_000;
+
+export interface VerifyPayload {
+  readonly definition: unknown;
+  readonly testPatch: string;
+  readonly goldPatch: string;
+}
+
+export type VerifyOutcome =
+  | {
+      readonly kind: "report";
+      readonly green: boolean;
+      readonly summary: string;
+      readonly rendered: string;
+      readonly remaining: number;
+    }
+  | { readonly kind: "error"; readonly message: string }
+  | { readonly kind: "exhausted" };
+
+/** Same formula as src/submission-hash.ts on the worker. */
+export function payloadHash(payload: VerifyPayload): string {
+  return createHash("sha256")
+    .update(`${JSON.stringify(payload.definition)}\0${payload.testPatch}\0${payload.goldPatch}`)
+    .digest("hex");
+}
+
+/**
+ * Sandbox side of the worker mailbox: writes a verify request the supervising activity picks up,
+ * then blocks until the response arrives. Reports consume the per-session budget; worker errors
+ * and budget exhaustion do not.
+ */
+export class VerifyClient {
+  readonly #mailbox = process.env.SELFBENCH_MAILBOX ?? DEFAULT_MAILBOX;
+  readonly #budget = Number(process.env.SELFBENCH_VERIFY_BUDGET ?? "0");
+  readonly #pollMs = Number(process.env.SELFBENCH_VERIFY_POLL_MS ?? String(DEFAULT_POLL_MS));
+  readonly #timeoutMs = Number(
+    process.env.SELFBENCH_VERIFY_TIMEOUT_MS ?? String(DEFAULT_TIMEOUT_MS),
+  );
+  #used = 0;
+  #sequence = 0;
+  #lastGreenHash: string | undefined;
+
+  get remaining(): number {
+    return Math.max(0, this.#budget - this.#used);
+  }
+
+  /** Whether this exact payload was verified green in this session. */
+  verifiedGreen(payload: VerifyPayload): boolean {
+    return this.#lastGreenHash !== undefined && this.#lastGreenHash === payloadHash(payload);
+  }
+
+  async verify(kind: "task" | "fix", payload: VerifyPayload): Promise<VerifyOutcome> {
+    if (this.remaining === 0) {
+      return { kind: "exhausted" };
+    }
+    const requests = join(this.#mailbox, "requests");
+    const responses = join(this.#mailbox, "responses");
+    mkdirSync(requests, { recursive: true });
+    mkdirSync(responses, { recursive: true });
+    this.#sequence += 1;
+    const id = `${Date.now()}-${this.#sequence}`;
+    const request = join(requests, `${id}.json`);
+    writeFileSync(`${request}.tmp`, `${JSON.stringify({ id, kind, ...payload })}\n`);
+    renameSync(`${request}.tmp`, request);
+    const response = join(responses, `${id}.json`);
+    const deadline = Date.now() + this.#timeoutMs;
+    while (Date.now() < deadline) {
+      if (existsSync(response)) {
+        let parsed: unknown;
+        try {
+          parsed = JSON.parse(readFileSync(response, "utf8"));
+        } catch {
+          parsed = undefined;
+        }
+        if (parsed && typeof parsed === "object") {
+          return this.#record(parsed as Record<string, unknown>, payload);
+        }
+      }
+      await new Promise((resolve) => setTimeout(resolve, this.#pollMs));
+    }
+    this.#used += 1;
+    return {
+      kind: "error",
+      message: `verify timed out after ${Math.round(this.#timeoutMs / 60_000)} minutes; the worker may still be building. ${this.remaining} verify call(s) remain.`,
+    };
+  }
+
+  #record(response: Record<string, unknown>, payload: VerifyPayload): VerifyOutcome {
+    if (response.kind !== "report") {
+      return {
+        kind: "error",
+        message:
+          typeof response.message === "string" ? response.message : "verify failed on the worker",
+      };
+    }
+    this.#used += 1;
+    const green = response.green === true;
+    if (green) {
+      this.#lastGreenHash = payloadHash(payload);
+    }
+    return {
+      kind: "report",
+      green,
+      summary: typeof response.summary === "string" ? response.summary : "",
+      rendered: typeof response.rendered === "string" ? response.rendered : "",
+      remaining: this.remaining,
+    };
+  }
+}
+
+export function verifyOutcomeText(outcome: VerifyOutcome, submitTool: string): string {
+  if (outcome.kind === "exhausted") {
+    return `No verify calls remain in this session. Call ${submitTool} with your best task or explain in your final message why it cannot be made fair.`;
+  }
+  if (outcome.kind === "error") {
+    return `verify could not complete: ${outcome.message}`;
+  }
+  const next = outcome.green
+    ? `Call ${submitTool} with exactly this payload to record it; the worker will reuse this report.`
+    : `Fix the red gates and call verify again, or ${submitTool} if you must (the worker verifies again and a round is spent if it fails).`;
+  return `${outcome.rendered.trim()}\n\nverify result: green=${outcome.green}. ${outcome.remaining} verify call(s) remain in this session. ${next}`;
+}