self-bench 0.3.0 → 0.3.3

package/docs/task-construction.md

@@ -0,0 +1,94 @@
+# Task construction and validation
+
+SelfBench creates easy, medium, and hard Harbor evaluations from completed pull requests. This document defines the task boundary, validation gates, repair behavior, and export contents.
+
+## Terms
+
+- A **candidate** is one possible source pull request.
+- The **base snapshot** is the repository state before the change.
+- The **reference patch** or **gold** is the known-good non-test implementation.
+- The **held-out test patch** is grader code hidden from evaluated agents.
+- A **fail-to-pass** test fails on the base snapshot and passes after a correct implementation.
+- A **pass-to-pass** test already passes at the base and protects existing behavior.
+- A Harbor **nop** run applies no solution; an **oracle** run applies the saved reference patch.
+- **Test-to-gold coupling** means a test passes only because it knows the saved implementation's private structure rather than the requested behavior.
+
+## Provenance
+
+Each task is grounded in one retained human request. SelfBench prefers exact user messages from local Pi, Claude Code, or Codex sessions associated with the repository or its worktrees. For third-party repositories, an exact GitHub pull-request title and optional body from a non-bot author is also valid.
+
+GitHub records are stored with a distinct `github-pull-request` source type, canonical URL, and PR number. Discovery cannot pair the request from one PR with another PR. A model-authored benchmark instruction may restate the retained request but may not add behavior inferred only from the implementation or tests.
+
+## Difficulty gates
+
+Profiles are eligibility rules, not empirical claims about model success:
+
+| Difficulty | Reference patch | Fail-to-pass | Pass-to-pass |
+| --- | --- | --- | --- |
+| easy | at least 20 changed lines across 1 implementation path | at least 1 | no minimum |
+| medium | at least 50 changed lines across 2 implementation paths | at least 1 | at least 1 |
+| hard | at least 100 changed lines across 3 implementation paths | at least 1 | at least 2 |
+
+Every accepted task also requires a held-out test patch with no file overlap with the reference patch, deterministic repository-native setup and tests, a passing nop/oracle split, and independent anti-coupling review.
+
+The size gate is mechanical. Generated or vendored code suitability remains a review judgment. Git LFS, submodules, generated changes, and service-heavy integration suites receive no special path and may be rejected during authoring or validation.
+
+## Agent-visible boundary
+
+An evaluated coding agent receives:
+
+- the base repository snapshot;
+- a standalone instruction preserving the human request;
+- the task's declared environment.
+
+It does not receive the held-out test patch or reference solution. Harbor mounts `solution/` only for the explicit oracle agent.
+
+Held-out tests must exercise an existing public API, command, persistence boundary, or extension seam. They may not import gold-specific private helpers or prescribe exact internal SQL, query counts, private schemas, object identity, telemetry layout, incidental error wording, or UI composition unless the source request explicitly makes that artifact public.
+
+## Validation and repair
+
+Static audit runs before sandbox validation. Harbor then proves:
+
+1. `nop`: selected new tests fail while selected regressions pass;
+2. `oracle`: the reference patch applies and every selected test passes;
+3. determinism: the fail-to-pass selection passes a second time with the oracle.
+
+A fresh model session reviews the prompt and tests without inheriting the authoring conversation. When it finds repairable test-to-gold coupling, Temporal schedules one constrained repair in another sandbox. Repair may modify only paths already changed by the held-out test patch. It cannot change the request, base snapshot, or reference implementation.
+
+After repair, the task repeats static audit, nop, oracle, and independent review from the beginning. A second failure rejects the candidate.
+
+## Toolchains
+
+The task compiler has setup layers for Node.js, Bun, Python, Go, and Rust. The author supplies fixed repository-native setup and test commands; compatibility is validated per candidate instead of promised for every repository.
+
+When the reference patch changes a recognized dependency manifest or lockfile, the hidden verifier image repeats setup with the trusted reference state and then resets source files to the base snapshot. This prevents stale base dependencies from invalidating the oracle without exposing the reference patch to the coding-agent environment.
+
+A test selector is one repository-native identifier substituted into the task's `{tests}` command template. For example, `bun test {tests}` may receive one new test path and two existing regression paths. Pytest, Go, Rust, and custom runners use their own selectors.
+
+## Export
+
+The run export is a gzip-compressed tar archive:
+
+```text
+manifest.json
+tasks/
+├── task-one.tar.gz
+└── task-two.tar.gz
+```
+
+`manifest.json` pins the source repository commit, SelfBench build identity, execution backend, task IDs, and archive SHA-256 values. Each task archive is a native Harbor task:
+
+```text
+harbor-task/
+├── task.toml
+├── instruction.md
+├── environment/
+├── tests/
+└── solution/
+```
+
+The export includes the repository snapshot at each selected base commit, held-out tests, and reference solutions. It excludes Git history, local provenance/session records, and author/reviewer transcripts.
+
+The manifest digest detects accidental corruption but is not a signature because it sits inside the same archive. Extract only exports from a trusted SelfBench deployment and store them as private benchmark material.
+
+The complete model-facing authoring and anti-coupling rubric is in [`src/skills/selfbench/SKILL.md`](../src/skills/selfbench/SKILL.md).

package/package.json

@@ -1,25 +1,38 @@
 {
   "name": "self-bench",
-  "version": "0.3.0",
+  "version": "0.3.3",
   "description": "Turn completed repository changes into durable, private Harbor evaluations",
   "license": "MIT",
   "repository": {
     "type": "git",
     "url": "https://github.com/mupt-ai/self-bench.git"
   },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
   "type": "module",
   "bin": {
-    "selfbench": "dist/cli.js",
-    "selfbench-agent-smoke": "dist/agent-smoke-main.js",
-    "selfbench-eval": "dist/eval-main.js",
-    "selfbench-reaudit": "dist/reaudit-main.js",
-    "selfbench-repair": "dist/repair-main.js",
-    "selfbench-validate": "dist/validate-main.js"
+    "self-bench": "dist/cli.js",
+    "self-bench-agent-smoke": "dist/agent-smoke-main.js",
+    "self-bench-eval": "dist/eval-main.js",
+    "self-bench-reaudit": "dist/reaudit-main.js",
+    "self-bench-repair": "dist/repair-main.js",
+    "self-bench-validate": "dist/validate-main.js"
   },
   "files": [
     "dist",
-    "src/extensions",
-    "src/skills"
+    "src",
+    "docs",
+    "compose.yaml",
+    "Dockerfile",
+    "Dockerfile.sandbox",
+    "bun.lock",
+    "scripts",
+    "biome.json",
+    "tsconfig.json",
+    "tsconfig.build.json",
+    ".dockerignore"
   ],
   "engines": {
     "node": ">=22"
@@ -29,44 +42,44 @@
     "clean": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"",
     "cli": "bun run src/cli.ts",
     "build:review": "vite build --config review/vite.config.ts",
-    "build:server": "tsc -p tsconfig.build.json && bun build src/sandbox-author.ts --target=node --outfile=dist/sandbox-author.bundle.js && bun build src/sandbox-review.ts --target=node --outfile=dist/sandbox-review.bundle.js && bun build src/sandbox-repair.ts --target=node --outfile=dist/sandbox-repair.bundle.js && bun build src/sandbox-validation-repair.ts --target=node --outfile=dist/sandbox-validation-repair.bundle.js",
+    "build:server": "tsc -p tsconfig.build.json && bun build src/sandbox-author.ts --target=node --outfile=dist/sandbox-author.bundle.js && bun build src/sandbox-review.ts --target=node --outfile=dist/sandbox-review.bundle.js && bun build src/sandbox-repair.ts --target=node --outfile=dist/sandbox-repair.bundle.js && bun build src/sandbox-validation-repair.ts --target=node --outfile=dist/sandbox-validation-repair.bundle.js && bun run build:metadata",
+    "build:metadata": "bun scripts/write-build-metadata.ts",
+    "prepack": "bun run build",
+    "prepublishOnly": "bun run validate",
+    "verify:package": "bun scripts/verify-package.ts",
     "check": "biome check . && tsc --noEmit -p tsconfig.json && bun run typecheck:review",
     "dev:api": "tsx src/api-main.ts",
     "dev:review": "vite --config review/vite.config.ts",
-    "dev:worker": "tsx src/worker-main.ts",
+    "dev:worker": "tsx src/temporal/worker-main.ts",
     "format": "biome format --write .",
-    "prepack": "bun run build",
     "start:api": "node dist/api-main.js",
-    "start:worker": "node dist/worker-main.js",
+    "start:worker": "node dist/temporal/worker-main.js",
     "test": "bun test tests review/src",
     "typecheck:review": "tsc --noEmit -p review/tsconfig.json",
-    "validate": "bun run check && bun run test && bun run build"
-  },
-  "publishConfig": {
-    "access": "public"
+    "validate": "bun run check && bun run test && bun run build && bun run verify:package"
   },
   "dependencies": {
     "@google-cloud/storage": "^7.17.0",
-    "@pierre/diffs": "1.2.12",
     "@temporalio/activity": "^1.13.2",
     "@temporalio/client": "^1.13.2",
     "@temporalio/common": "^1.13.2",
     "@temporalio/worker": "^1.13.2",
     "@temporalio/workflow": "^1.13.2",
     "modal": "^0.9.0",
-    "react": "19.2.7",
-    "react-dom": "19.2.7",
     "zod": "^4.1.12"
   },
   "devDependencies": {
     "@biomejs/biome": "^2.2.6",
     "@earendil-works/pi-coding-agent": "0.84.0",
+    "@pierre/diffs": "1.2.12",
     "@sinclair/typebox": "0.34.41",
     "@types/bun": "^1.3.14",
     "@types/node": "24.10.13",
     "@types/react": "19.2.14",
     "@types/react-dom": "19.2.3",
     "@vitejs/plugin-react": "4.7.0",
+    "react": "19.2.7",
+    "react-dom": "19.2.7",
     "tsx": "^4.20.6",
     "typescript": "5.9.3",
     "vite": "7.3.1"

package/scripts/verify-package.ts

@@ -0,0 +1,57 @@
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+
+const root = resolve(import.meta.dir, "..");
+const temporary = await mkdtemp(join(tmpdir(), "self-bench-package-"));
+async function run(command: string, args: string[], cwd = root): Promise<string> {
+  const child = Bun.spawn([command, ...args], {
+    cwd,
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const [exitCode, stdout, stderr] = await Promise.all([
+    child.exited,
+    new Response(child.stdout).text(),
+    new Response(child.stderr).text(),
+  ]);
+  if (exitCode !== 0) {
+    throw new Error(`${command} ${args.join(" ")} failed:\n${stderr || stdout}`);
+  }
+  return stdout;
+}
+
+try {
+  await run("bun", ["pm", "pack", "--destination", temporary]);
+  const tarballs = (await Array.fromAsync(new Bun.Glob("*.tgz").scan({ cwd: temporary }))).map(
+    (name) => join(temporary, name),
+  );
+  const tarball = tarballs[0] ?? (() => {
+    throw new Error("bun pm pack did not create a tarball");
+  })();
+  const packageJson = JSON.parse(await readFile(join(root, "package.json"), "utf8")) as {
+    name: string;
+    version: string;
+  };
+  const installRoot = join(temporary, "install");
+  await run("mkdir", ["-p", installRoot]);
+  await run("bun", ["init", "--yes"], installRoot);
+  await run("bun", ["add", "--no-save", tarball], installRoot);
+  const executable = join(installRoot, "node_modules", ".bin", "self-bench");
+  const help = await run(executable, ["--help"], installRoot);
+  if (!help.includes("self-bench up")) {
+    throw new Error("installed self-bench did not print the expected CLI help");
+  }
+  for (const asset of ["compose.yaml", "Dockerfile", "Dockerfile.sandbox", "src/skills/selfbench/SKILL.md"]) {
+    await readFile(join(installRoot, "node_modules", packageJson.name, asset));
+  }
+  const installedPackage = JSON.parse(
+    await readFile(join(installRoot, "node_modules", packageJson.name, "package.json"), "utf8"),
+  ) as { name: string; version: string };
+  if (installedPackage.name !== packageJson.name || installedPackage.version !== packageJson.version) {
+    throw new Error("installed package metadata does not match the workspace package");
+  }
+  console.log(`verified ${installedPackage.name}@${installedPackage.version}`);
+} finally {
+  await rm(temporary, { recursive: true, force: true });
+}

package/scripts/write-build-metadata.ts

@@ -0,0 +1,27 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+const execFileAsync = promisify(execFile);
+const root = new URL("../", import.meta.url);
+const output = new URL("../dist/build-metadata.js", import.meta.url);
+const configuredCommit = process.env.SELFBENCH_BUILD_COMMIT;
+
+let commit = configuredCommit;
+if (!commit) {
+  try {
+    const result = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root });
+    commit = result.stdout.trim();
+  } catch {
+    commit = "0".repeat(40);
+  }
+}
+if (!/^[0-9a-f]{40}$/i.test(commit)) {
+  throw new Error("SELFBENCH_BUILD_COMMIT must be a full commit SHA");
+}
+
+await mkdir(new URL("../dist/", import.meta.url), { recursive: true });
+await writeFile(
+  output,
+  `// Generated during the build.\nexport const buildCommit = ${JSON.stringify(commit.toLowerCase())};\n`,
+);

package/src/agent-smoke-main.ts

@@ -0,0 +1,63 @@
+#!/usr/bin/env node
+
+import { parseArgs } from "node:util";
+import { smokeAllAdapters } from "./agent-smoke.js";
+
+const parsed = parseArgs({
+  options: {
+    task: { type: "string" },
+    jobs: { type: "string" },
+    harbor: { type: "string" },
+    environment: { type: "string", default: "modal" },
+    concurrency: { type: "string", default: "4" },
+    help: { type: "boolean", short: "h" },
+  },
+  strict: true,
+});
+if (parsed.values.help) {
+  console.log(`Attempt installation and instantiation of every pinned Harbor adapter.
+
+Usage:
+  self-bench-agent-smoke --task DIRECTORY --jobs DIRECTORY [options]
+
+Options:
+  --harbor PATH              Harbor executable (default: harbor)
+  --environment docker|modal Execution environment (default: modal)
+  --concurrency N            Concurrent adapter checks (default: 4)
+  -h, --help                 Show this help`);
+  process.exit(0);
+}
+const environment = parsed.values.environment;
+if (environment !== "docker" && environment !== "modal") {
+  throw new Error("--environment must be docker or modal");
+}
+const results = await smokeAllAdapters({
+  taskDirectory: parsed.values.task ?? fail("--task is required"),
+  jobsDirectory: parsed.values.jobs ?? fail("--jobs is required"),
+  environment,
+  concurrency: positiveInteger(parsed.values.concurrency, "--concurrency"),
+  ...(parsed.values.harbor ? { harborPath: parsed.values.harbor } : {}),
+});
+console.log(
+  JSON.stringify(
+    {
+      adapters: results.length,
+      installed: results.filter((result) => result.installed).length,
+      failed: results.filter((result) => !result.installed).length,
+    },
+    null,
+    2,
+  ),
+);
+
+function fail(message: string): never {
+  throw new Error(message);
+}
+
+function positiveInteger(value: string | undefined, label: string): number {
+  const parsed = Number(value);
+  if (!Number.isInteger(parsed) || parsed < 1) {
+    throw new Error(`${label} must be a positive integer`);
+  }
+  return parsed;
+}

package/src/agent-smoke.ts

@@ -0,0 +1,132 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { parallelMap } from "./parallel.js";
+import { runCommand } from "./process.js";
+
+export const HARBOR_AGENT_ADAPTERS = [
+  "oracle",
+  "nop",
+  "acp",
+  "terminus-2",
+  "claude-code",
+  "copilot-cli",
+  "aider",
+  "cline-cli",
+  "codex",
+  "cortex-code",
+  "cursor-cli",
+  "gemini-cli",
+  "antigravity-cli",
+  "antigravity-sdk",
+  "rovodev-cli",
+  "goose",
+  "grok-build",
+  "hermes",
+  "kimi-code",
+  "kimi-cli",
+  "langgraph",
+  "deerflow",
+  "mini-swe-agent",
+  "nemo-agent",
+  "swe-agent",
+  "opencode",
+  "mimo",
+  "openclaw",
+  "openhands",
+  "openhands-sdk",
+  "pi",
+  "qwen-coder",
+  "devin",
+  "trae-agent",
+  "computer-1",
+  "eve",
+  "dspy-rlm",
+  "vibe",
+] as const;
+
+export interface AdapterSmokeOptions {
+  readonly taskDirectory: string;
+  readonly jobsDirectory: string;
+  readonly harborPath?: string;
+  readonly environment?: "docker" | "modal";
+  readonly concurrency?: number;
+}
+
+export interface AdapterSmokeResult {
+  readonly agent: (typeof HARBOR_AGENT_ADAPTERS)[number];
+  readonly installed: boolean;
+  readonly exitCode: number;
+  readonly jobName: string;
+  readonly outputTail: string;
+}
+
+export async function smokeAllAdapters(
+  options: AdapterSmokeOptions,
+): Promise<readonly AdapterSmokeResult[]> {
+  const jobsDirectory = resolve(options.jobsDirectory);
+  const reportsDirectory = join(jobsDirectory, "adapter-smoke");
+  await mkdir(reportsDirectory, { recursive: true });
+  const results = await parallelMap(
+    HARBOR_AGENT_ADAPTERS,
+    options.concurrency ?? 4,
+    async (agent) => {
+      const reportPath = join(reportsDirectory, `${agent}.json`);
+      const existing = await readFile(reportPath, "utf8").catch(() => undefined);
+      if (existing) {
+        return JSON.parse(existing) as AdapterSmokeResult;
+      }
+      const jobName = `install-${agent}-${crypto.randomUUID().slice(0, 8)}`;
+      const result = await runCommand(
+        options.harborPath ?? "harbor",
+        [
+          "run",
+          "--path",
+          resolve(options.taskDirectory),
+          "--agent",
+          agent,
+          "--env",
+          options.environment ?? "modal",
+          "--job-name",
+          jobName,
+          "--jobs-dir",
+          jobsDirectory,
+          "--install-only",
+          "--n-concurrent",
+          "1",
+          "--max-retries",
+          "0",
+          "--delete",
+          "--yes",
+          "--quiet",
+        ],
+        { allowFailure: true, timeoutMs: 60 * 60 * 1000 },
+      );
+      const summary: AdapterSmokeResult = {
+        agent,
+        installed: result.exitCode === 0,
+        exitCode: result.exitCode,
+        jobName,
+        outputTail: `${result.stdout}\n${result.stderr}`.trim().slice(-2_000),
+      };
+      await writeFile(reportPath, `${JSON.stringify(summary, null, 2)}\n`, { flag: "wx" });
+      return summary;
+    },
+  );
+  await writeFile(
+    join(reportsDirectory, "summary.json"),
+    `${JSON.stringify(
+      {
+        schemaVersion: 1,
+        mode: "install-only",
+        environment: options.environment ?? "modal",
+        adapterCount: results.length,
+        installed: results.filter((result) => result.installed).length,
+        failed: results.filter((result) => !result.installed).length,
+        results,
+      },
+      null,
+      2,
+    )}\n`,
+  );
+  return results;
+}

package/src/api-main.ts

@@ -0,0 +1,12 @@
+import { startApi } from "./api.js";
+import { loadConfig } from "./config.js";
+
+const config = loadConfig();
+const stop = await startApi(config);
+console.log(`SelfBench API listening on http://${config.apiHost}:${config.apiPort}`);
+
+for (const signal of ["SIGINT", "SIGTERM"] as const) {
+  process.once(signal, () => {
+    void stop().finally(() => process.exit(0));
+  });
+}