self-bench 0.3.6 → 0.3.8

package/review/src/export-loader.ts

@@ -1,151 +0,0 @@
-import type { ExportManifest, ExportTask, LoadedExport } from "./types";
-
-const textNames = new Set([
-  "definition.json",
-  "instruction.md",
-  "task.toml",
-  ".selfbench-manifest.json",
-  "solution/gold.patch",
-  "tests/test.patch",
-  "tests/dependency-setup.patch",
-]);
-
-interface TarEntry {
-  name: string;
-  data: Uint8Array;
-}
-
-export async function loadExport(input: Blob | Uint8Array): Promise<LoadedExport> {
-  const archive = input instanceof Blob ? new Uint8Array(await input.arrayBuffer()) : input;
-  const outerEntries = untar(await gunzip(archive));
-  const manifestBytes = outerEntries.find((entry) => entry.name === "manifest.json")?.data;
-  if (!manifestBytes) throw new Error("export is missing manifest.json");
-  const manifest = JSON.parse(new TextDecoder().decode(manifestBytes)) as ExportManifest;
-  const tasks: ExportTask[] = [];
-  const taskArchives = new Map<string, Uint8Array>();
-  for (const manifestTask of manifest.tasks) {
-    const name = `tasks/${manifestTask.taskId}.tar.gz`;
-    const archiveEntry = outerEntries.find((entry) => entry.name === name);
-    if (!archiveEntry) throw new Error(`export is missing ${name}`);
-    taskArchives.set(manifestTask.taskId, archiveEntry.data);
-    const entries = untar(await gunzip(archiveEntry.data));
-    const files = new Map<string, Uint8Array>();
-    const textFiles = new Map<string, string>();
-    for (const entry of entries) {
-      const relative = entry.name.replace(/^harbor-task\//, "");
-      if (!relative || entry.name.endsWith("/")) continue;
-      files.set(relative, entry.data);
-      if (textNames.has(relative) || relative.endsWith(".json")) {
-        textFiles.set(relative, new TextDecoder().decode(entry.data));
-      }
-    }
-    tasks.push({ taskId: manifestTask.taskId, files, textFiles });
-  }
-  return { manifest, tasks, taskArchives };
-}
-
-export async function buildCleanExport(
-  loaded: LoadedExport,
-  removedTaskIds: Set<string>,
-): Promise<Blob> {
-  const keptIds = new Set(
-    loaded.tasks.filter((task) => !removedTaskIds.has(task.taskId)).map((task) => task.taskId),
-  );
-  const manifest: ExportManifest = {
-    ...loaded.manifest,
-    acceptedCount: keptIds.size,
-    tasks: loaded.manifest.tasks.filter((task) => keptIds.has(task.taskId)),
-  };
-  const entries: TarEntry[] = [
-    {
-      name: "manifest.json",
-      data: new TextEncoder().encode(`${JSON.stringify(manifest, null, 2)}\n`),
-    },
-  ];
-  for (const task of loaded.tasks) {
-    if (keptIds.has(task.taskId)) {
-      const archive = loaded.taskArchives.get(task.taskId);
-      if (!archive) throw new Error(`loaded export is missing ${task.taskId}`);
-      entries.push({ name: `tasks/${task.taskId}.tar.gz`, data: archive });
-    }
-  }
-  return gzip(tar(entries));
-}
-
-async function gunzip(value: Uint8Array): Promise<Uint8Array> {
-  return transform(value, new DecompressionStream("gzip"));
-}
-
-async function gzip(value: Uint8Array): Promise<Blob> {
-  const compressed = await transform(value, new CompressionStream("gzip"));
-  return new Blob([compressed.buffer as ArrayBuffer], { type: "application/gzip" });
-}
-
-async function transform(value: Uint8Array, stream: TransformStream): Promise<Uint8Array> {
-  const output = await new Response(
-    new Blob([value.buffer as ArrayBuffer]).stream().pipeThrough(stream),
-  ).arrayBuffer();
-  return new Uint8Array(output);
-}
-
-function untar(buffer: Uint8Array): TarEntry[] {
-  const entries: TarEntry[] = [];
-  for (let offset = 0; offset + 512 <= buffer.length; ) {
-    const header = buffer.subarray(offset, offset + 512);
-    if (header.every((byte) => byte === 0)) break;
-    const name = readString(header, 0, 100);
-    const size = parseOctal(header, 124, 12);
-    const type = header[156];
-    offset += 512;
-    const data = buffer.slice(offset, offset + size);
-    if (type !== 5 && name) entries.push({ name, data });
-    offset += Math.ceil(size / 512) * 512;
-  }
-  return entries;
-}
-
-function tar(entries: TarEntry[]): Uint8Array {
-  const chunks: Uint8Array[] = [];
-  for (const entry of entries) {
-    const header = new Uint8Array(512);
-    writeString(header, 0, 100, entry.name);
-    writeOctal(header, 100, 8, 0o644);
-    writeOctal(header, 108, 8, 0);
-    writeOctal(header, 116, 8, 0);
-    writeOctal(header, 124, 12, entry.data.length);
-    writeOctal(header, 136, 12, Math.floor(Date.now() / 1000));
-    header.fill(32, 148, 156);
-    header[156] = 48;
-    writeString(header, 257, 6, "ustar\0");
-    writeString(header, 263, 2, "00");
-    const checksum = header.reduce((sum, byte) => sum + byte, 0);
-    writeString(header, 148, 8, `${checksum.toString(8).padStart(6, "0")}\0 `);
-    chunks.push(header, entry.data, new Uint8Array((512 - (entry.data.length % 512)) % 512));
-  }
-  chunks.push(new Uint8Array(1024));
-  const output = new Uint8Array(chunks.reduce((sum, chunk) => sum + chunk.length, 0));
-  let offset = 0;
-  for (const chunk of chunks) {
-    output.set(chunk, offset);
-    offset += chunk.length;
-  }
-  return output;
-}
-
-function readString(bytes: Uint8Array, start: number, length: number): string {
-  const end = bytes.indexOf(0, start);
-  return new TextDecoder().decode(bytes.subarray(start, end < 0 ? start + length : end));
-}
-
-function parseOctal(bytes: Uint8Array, start: number, length: number): number {
-  const value = readString(bytes, start, length).trim();
-  return value ? Number.parseInt(value, 8) : 0;
-}
-
-function writeString(bytes: Uint8Array, start: number, length: number, value: string): void {
-  bytes.set(new TextEncoder().encode(value).subarray(0, length), start);
-}
-
-function writeOctal(bytes: Uint8Array, start: number, length: number, value: number): void {
-  writeString(bytes, start, length, value.toString(8).padStart(length - 1, "0"));
-}

package/review/src/main.tsx

@@ -1,15 +0,0 @@
-import React from "react";
-import { createRoot } from "react-dom/client";
-import { App } from "./App";
-import "./styles.css";
-
-const root = document.getElementById("root");
-if (!root) {
-  throw new Error("review root is missing");
-}
-
-createRoot(root).render(
-  <React.StrictMode>
-    <App />
-  </React.StrictMode>,
-);

package/review/src/styles.css

@@ -1,162 +0,0 @@
-:root {
-  --bg: #f4f5f7;
-  --surface: #ffffff;
-  --border: #d8dce3;
-  --text: #1a1d23;
-  --muted: #6b7280;
-  --accent: #2563eb;
-  --danger: #dc2626;
-  --success: #059669;
-  --code-bg: #f1f3f5;
-  --font: -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
-  --mono: "SF Mono", "Cascadia Code", "Fira Code", monospace;
-}
-
-@media (prefers-color-scheme: dark) {
-  :root {
-    --bg: #0d1117;
-    --surface: #161b22;
-    --border: #30363d;
-    --text: #e6edf3;
-    --muted: #8b949e;
-    --accent: #58a6ff;
-    --danger: #f85149;
-    --success: #3fb950;
-    --code-bg: #1c2128;
-  }
-}
-
-* { margin: 0; padding: 0; box-sizing: border-box; }
-
-body {
-  font-family: var(--font);
-  background: var(--bg);
-  color: var(--text);
-  font-size: 14px;
-  line-height: 1.5;
-}
-
-.app-shell { min-height: 100vh; display: flex; flex-direction: column; }
-
-.topbar {
-  display: flex; justify-content: space-between; align-items: center;
-  padding: 16px 24px; border-bottom: 1px solid var(--border);
-  background: var(--surface);
-}
-
-.eyebrow {
-  font-size: 11px; font-weight: 600; letter-spacing: 0.08em;
-  text-transform: uppercase; color: var(--muted);
-}
-
-.topbar h1 { font-size: 18px; font-weight: 600; margin-top: 2px; }
-.topbar-actions { display: flex; gap: 8px; align-items: center; }
-.token-input {
-  padding: 6px 10px; border: 1px solid var(--border); border-radius: 6px;
-  background: var(--surface); color: var(--text); font-size: 12px; width: 180px;
-}
-
-.error-banner {
-  padding: 12px 24px; background: var(--danger); color: #fff; font-size: 13px;
-}
-
-.workspace {
-  flex: 1; display: grid; grid-template-columns: 280px 1fr; gap: 16px;
-  padding: 16px 24px;
-}
-
-.surface { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; }
-
-.run-list { padding: 16px; overflow-y: auto; max-height: calc(100vh - 100px); }
-
-.section-heading {
-  display: flex; justify-content: space-between; align-items: baseline;
-  margin-bottom: 12px; padding-bottom: 8px; border-bottom: 1px solid var(--border);
-}
-
-.section-heading h2 { font-size: 13px; font-weight: 600; }
-.section-heading span { font-size: 11px; color: var(--muted); }
-
-.run-row {
-  width: 100%; display: flex; flex-direction: column; gap: 2px;
-  padding: 10px 12px; margin-bottom: 6px; border: 1px solid var(--border);
-  border-radius: 6px; background: var(--surface); cursor: pointer; text-align: left;
-  font: inherit; color: inherit;
-}
-
-.run-row:hover { border-color: var(--accent); }
-.run-row.selected { border-color: var(--accent); box-shadow: 0 0 0 1px var(--accent); }
-.run-row strong { font-size: 13px; }
-.run-row span { font-size: 11px; color: var(--muted); }
-.run-row small { font-size: 10px; color: var(--muted); }
-
-.content { display: flex; flex-direction: column; gap: 16px; }
-
-.empty {
-  padding: 64px 32px; text-align: center; display: flex; flex-direction: column;
-  align-items: center; gap: 8px; min-height: 400px; justify-content: center;
-}
-.empty h2 { font-size: 16px; font-weight: 600; }
-.empty p { color: var(--muted); }
-
-.summary {
-  display: flex; justify-content: space-between; align-items: center;
-  padding: 16px 20px;
-}
-.summary h2 { font-size: 16px; font-weight: 600; }
-.summary .muted { font-size: 12px; color: var(--muted); margin-top: 2px; }
-
-.review-grid { display: grid; grid-template-columns: 240px 1fr; gap: 16px; }
-
-.task-list { padding: 16px; max-height: calc(100vh - 280px); overflow-y: auto; }
-
-.task-row {
-  width: 100%; display: flex; justify-content: space-between; align-items: center;
-  padding: 8px 10px; margin-bottom: 4px; border: 1px solid var(--border);
-  border-radius: 5px; background: var(--surface); cursor: pointer; text-align: left;
-  font: inherit; color: inherit; font-size: 12px;
-}
-.task-row:hover { border-color: var(--accent); }
-.task-row.selected { border-color: var(--accent); }
-.task-row.removed { opacity: 0.5; }
-.task-row.removed strong { text-decoration: line-through; }
-.task-row span { font-size: 10px; color: var(--muted); }
-
-.task-detail { padding: 20px 24px; max-height: calc(100vh - 280px); overflow-y: auto; }
-
-.detail-header {
-  display: flex; justify-content: space-between; align-items: flex-start;
-  margin-bottom: 20px; padding-bottom: 16px; border-bottom: 1px solid var(--border);
-}
-.detail-header h2 { font-size: 18px; font-weight: 600; }
-
-.detail-block { margin-bottom: 20px; }
-.detail-block h3 {
-  font-size: 12px; font-weight: 600; letter-spacing: 0.04em;
-  text-transform: uppercase; color: var(--muted); margin-bottom: 6px;
-}
-.detail-block pre {
-  padding: 12px 16px; background: var(--code-bg); border-radius: 6px;
-  font-family: var(--mono); font-size: 12px; white-space: pre-wrap;
-  word-break: break-all; max-height: 400px; overflow-y: auto; line-height: 1.6;
-}
-.patch-files {
-  display: grid; gap: 10px; padding: 8px; max-height: 600px; overflow: auto;
-  background: var(--code-bg); border-radius: 6px;
-}
-.patch-files > div { overflow: hidden; border: 1px solid var(--border); border-radius: 4px; }
-
-.button {
-  padding: 7px 14px; border: 1px solid var(--border); border-radius: 6px;
-  background: var(--surface); color: var(--text); font-size: 12px; font-weight: 500;
-  cursor: pointer; font-family: var(--font);
-}
-.button:hover { border-color: var(--accent); color: var(--accent); }
-.button.primary { background: var(--accent); color: #fff; border-color: var(--accent); }
-.button.primary:hover { opacity: 0.9; color: #fff; }
-.button.danger { color: var(--danger); border-color: var(--danger); }
-.button.danger:hover { background: var(--danger); color: #fff; }
-.button.undo { color: var(--success); border-color: var(--success); }
-.button.undo:hover { background: var(--success); color: #fff; }
-
-.muted { color: var(--muted); }

package/review/src/types.ts

@@ -1,47 +0,0 @@
-export interface RunSummary {
-  runId: string;
-  status: string;
-  startedAt: string;
-  closedAt?: string;
-}
-
-export interface TaskProgress {
-  taskId: string;
-  candidateId: string;
-  difficulty: "easy" | "medium" | "hard";
-  status: string;
-  reason?: string;
-}
-
-export interface RunStatus {
-  runId: string;
-  phase: string;
-  requested: number;
-  accepted: number;
-  rejected: number;
-  tasks: TaskProgress[];
-  export?: { uri: string; sizeBytes: number; contentType: string };
-  error?: string;
-}
-
-export interface ExportManifest {
-  schemaVersion: number;
-  runId: string;
-  candidateCounts: Record<string, number>;
-  repository: { url: string; commit: string };
-  version: Record<string, unknown>;
-  acceptedCount: number;
-  tasks: { taskId: string; sha256: string }[];
-}
-
-export interface ExportTask {
-  taskId: string;
-  files: Map<string, Uint8Array>;
-  textFiles: Map<string, string>;
-}
-
-export interface LoadedExport {
-  manifest: ExportManifest;
-  tasks: ExportTask[];
-  taskArchives: Map<string, Uint8Array>;
-}

package/review/tsconfig.json

@@ -1,16 +0,0 @@
-{
-  "compilerOptions": {
-    "allowSyntheticDefaultImports": true,
-    "esModuleInterop": true,
-    "jsx": "react-jsx",
-    "lib": ["DOM", "DOM.Iterable", "ES2023"],
-    "module": "ESNext",
-    "moduleResolution": "Bundler",
-    "noEmit": true,
-    "skipLibCheck": true,
-    "strict": true,
-    "target": "ES2022",
-    "types": ["bun", "vite/client"]
-  },
-  "include": ["src", "vite.config.ts"]
-}

package/review/vite.config.ts

@@ -1,20 +0,0 @@
-import react from "@vitejs/plugin-react";
-import { defineConfig } from "vite";
-
-export default defineConfig({
-  root: "review",
-  plugins: [react()],
-  build: {
-    outDir: "../dist/review",
-    emptyOutDir: true,
-  },
-  server: {
-    allowedHosts: ["avyays-mac-mini.tailf3cee5.ts.net"],
-    proxy: {
-      "/v1": "http://127.0.0.1:8080",
-    },
-  },
-  preview: {
-    allowedHosts: ["avyays-mac-mini.tailf3cee5.ts.net"],
-  },
-});

package/scripts/build-sandbox-programs.ts

@@ -1,22 +0,0 @@
-import { mkdir } from "node:fs/promises";
-import { join, resolve } from "node:path";
-
-const root = resolve(import.meta.dir, "..");
-const outputDirectory = join(root, "dist");
-
-const programs = ["author", "review", "repair", "validation-repair"] as const;
-
-await mkdir(outputDirectory, { recursive: true });
-await Promise.all(
-  programs.map(async (program) => {
-    const result = await Bun.build({
-      entrypoints: [join(root, "src/sandbox/programs", `${program}.ts`)],
-      outdir: outputDirectory,
-      naming: `sandbox-${program}.bundle.js`,
-      target: "node",
-    });
-    if (!result.success) {
-      throw new AggregateError(result.logs, `failed to bundle sandbox ${program} program`);
-    }
-  }),
-);

package/scripts/verify-package.ts

@@ -1,102 +0,0 @@
-import { mkdtemp, readFile, rm } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join, resolve } from "node:path";
-
-const options = process.argv.slice(2);
-const dockerBuild = options.length === 1 && options[0] === "--docker-build";
-if (options.length > 0 && !dockerBuild) {
-  throw new Error(`unknown verify-package argument: ${options.join(" ")}`);
-}
-const root = resolve(import.meta.dir, "..");
-const temporary = await mkdtemp(join(tmpdir(), "self-bench-package-"));
-
-async function run(command: string, args: string[], cwd = root): Promise<string> {
-  const child = Bun.spawn([command, ...args], {
-    cwd,
-    stdout: "pipe",
-    stderr: "pipe",
-  });
-  const [exitCode, stdout, stderr] = await Promise.all([
-    child.exited,
-    new Response(child.stdout).text(),
-    new Response(child.stderr).text(),
-  ]);
-  if (exitCode !== 0) {
-    throw new Error(`${command} ${args.join(" ")} failed:\n${stderr || stdout}`);
-  }
-  return stdout;
-}
-
-try {
-  await run("bun", ["pm", "pack", "--destination", temporary]);
-  const tarballs = (await Array.fromAsync(new Bun.Glob("*.tgz").scan({ cwd: temporary }))).map(
-    (name) => join(temporary, name),
-  );
-  const tarball =
-    tarballs[0] ??
-    (() => {
-      throw new Error("bun pm pack did not create a tarball");
-    })();
-  const packageJson = JSON.parse(await readFile(join(root, "package.json"), "utf8")) as {
-    name: string;
-    version: string;
-    bin: Record<string, string>;
-  };
-  const installRoot = join(temporary, "install");
-  await run("mkdir", ["-p", installRoot]);
-  await run("bun", ["init", "--yes"], installRoot);
-  await run("bun", ["add", "--no-save", tarball], installRoot);
-  const executable = join(installRoot, "node_modules", ".bin", "self-bench");
-  const help = await run(executable, ["--help"], installRoot);
-  if (!help.includes("self-bench up")) {
-    throw new Error("installed self-bench did not print the expected CLI help");
-  }
-  const installedRoot = join(installRoot, "node_modules", packageJson.name);
-  for (const [name, target] of Object.entries(packageJson.bin)) {
-    await readFile(join(installedRoot, target));
-    await readFile(join(installRoot, "node_modules", ".bin", name));
-  }
-  for (const asset of [
-    "dist/api-main.js",
-    "dist/temporal/worker-main.js",
-    "dist/sandbox-author.bundle.js",
-    "dist/sandbox-review.bundle.js",
-    "dist/sandbox-repair.bundle.js",
-    "dist/sandbox-validation-repair.bundle.js",
-    "dist/review/index.html",
-  ]) {
-    await readFile(join(installedRoot, asset));
-  }
-  for (const asset of [
-    ".dockerignore",
-    "compose.yaml",
-    "Dockerfile",
-    "Dockerfile.sandbox",
-    "review/index.html",
-    "review/src/App.tsx",
-    "review/vite.config.ts",
-    "src/extensions/authoring.ts",
-    "src/skills/selfbench/SKILL.md",
-  ]) {
-    await readFile(join(installedRoot, asset));
-  }
-  const installedPackage = JSON.parse(
-    await readFile(join(installedRoot, "package.json"), "utf8"),
-  ) as { name: string; version: string };
-  if (
-    installedPackage.name !== packageJson.name ||
-    installedPackage.version !== packageJson.version
-  ) {
-    throw new Error("installed package metadata does not match the workspace package");
-  }
-  if (dockerBuild) {
-    await run(
-      "docker",
-      ["build", "--target", "build", "--file", join(installedRoot, "Dockerfile"), installedRoot],
-      installedRoot,
-    );
-  }
-  console.log(`verified ${installedPackage.name}@${installedPackage.version}`);
-} finally {
-  await rm(temporary, { recursive: true, force: true });
-}

package/scripts/write-build-metadata.ts

@@ -1,27 +0,0 @@
-import { execFile } from "node:child_process";
-import { mkdir, writeFile } from "node:fs/promises";
-import { promisify } from "node:util";
-
-const execFileAsync = promisify(execFile);
-const root = new URL("../", import.meta.url);
-const output = new URL("../dist/build-metadata.js", import.meta.url);
-const configuredCommit = process.env.SELFBENCH_BUILD_COMMIT;
-
-let commit = configuredCommit;
-if (!commit) {
-  try {
-    const result = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root });
-    commit = result.stdout.trim();
-  } catch {
-    commit = "0".repeat(40);
-  }
-}
-if (!/^[0-9a-f]{40}$/i.test(commit)) {
-  throw new Error("SELFBENCH_BUILD_COMMIT must be a full commit SHA");
-}
-
-await mkdir(new URL("../dist/", import.meta.url), { recursive: true });
-await writeFile(
-  output,
-  `// Generated during the build.\nexport const buildCommit = ${JSON.stringify(commit.toLowerCase())};\n`,
-);

package/src/agent-smoke-main.ts

@@ -1,64 +0,0 @@
-#!/usr/bin/env node
-
-import { parseArgs } from "node:util";
-import { smokeAllAdapters } from "./agent-smoke.js";
-import { isHarborEnvironment } from "./providers.js";
-
-const parsed = parseArgs({
-  options: {
-    task: { type: "string" },
-    jobs: { type: "string" },
-    harbor: { type: "string" },
-    environment: { type: "string", default: "modal" },
-    concurrency: { type: "string", default: "4" },
-    help: { type: "boolean", short: "h" },
-  },
-  strict: true,
-});
-if (parsed.values.help) {
-  console.log(`Attempt installation and instantiation of every pinned Harbor adapter.
-
-Usage:
-  self-bench-agent-smoke --task DIRECTORY --jobs DIRECTORY [options]
-
-Options:
-  --harbor PATH              Harbor executable (default: harbor)
-  --environment docker|modal Execution environment (default: modal)
-  --concurrency N            Concurrent adapter checks (default: 4)
-  -h, --help                 Show this help`);
-  process.exit(0);
-}
-const environment = parsed.values.environment ?? "modal";
-if (!isHarborEnvironment(environment)) {
-  throw new Error("--environment must be docker or modal");
-}
-const results = await smokeAllAdapters({
-  taskDirectory: parsed.values.task ?? fail("--task is required"),
-  jobsDirectory: parsed.values.jobs ?? fail("--jobs is required"),
-  environment,
-  concurrency: positiveInteger(parsed.values.concurrency, "--concurrency"),
-  ...(parsed.values.harbor ? { harborPath: parsed.values.harbor } : {}),
-});
-console.log(
-  JSON.stringify(
-    {
-      adapters: results.length,
-      installed: results.filter((result) => result.installed).length,
-      failed: results.filter((result) => !result.installed).length,
-    },
-    null,
-    2,
-  ),
-);
-
-function fail(message: string): never {
-  throw new Error(message);
-}
-
-function positiveInteger(value: string | undefined, label: string): number {
-  const parsed = Number(value);
-  if (!Number.isInteger(parsed) || parsed < 1) {
-    throw new Error(`${label} must be a positive integer`);
-  }
-  return parsed;
-}