self-bench 0.3.5 → 0.3.8

package/review/src/styles.css

@@ -1,162 +0,0 @@
-:root {
-  --bg: #f4f5f7;
-  --surface: #ffffff;
-  --border: #d8dce3;
-  --text: #1a1d23;
-  --muted: #6b7280;
-  --accent: #2563eb;
-  --danger: #dc2626;
-  --success: #059669;
-  --code-bg: #f1f3f5;
-  --font: -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
-  --mono: "SF Mono", "Cascadia Code", "Fira Code", monospace;
-}
-
-@media (prefers-color-scheme: dark) {
-  :root {
-    --bg: #0d1117;
-    --surface: #161b22;
-    --border: #30363d;
-    --text: #e6edf3;
-    --muted: #8b949e;
-    --accent: #58a6ff;
-    --danger: #f85149;
-    --success: #3fb950;
-    --code-bg: #1c2128;
-  }
-}
-
-* { margin: 0; padding: 0; box-sizing: border-box; }
-
-body {
-  font-family: var(--font);
-  background: var(--bg);
-  color: var(--text);
-  font-size: 14px;
-  line-height: 1.5;
-}
-
-.app-shell { min-height: 100vh; display: flex; flex-direction: column; }
-
-.topbar {
-  display: flex; justify-content: space-between; align-items: center;
-  padding: 16px 24px; border-bottom: 1px solid var(--border);
-  background: var(--surface);
-}
-
-.eyebrow {
-  font-size: 11px; font-weight: 600; letter-spacing: 0.08em;
-  text-transform: uppercase; color: var(--muted);
-}
-
-.topbar h1 { font-size: 18px; font-weight: 600; margin-top: 2px; }
-.topbar-actions { display: flex; gap: 8px; align-items: center; }
-.token-input {
-  padding: 6px 10px; border: 1px solid var(--border); border-radius: 6px;
-  background: var(--surface); color: var(--text); font-size: 12px; width: 180px;
-}
-
-.error-banner {
-  padding: 12px 24px; background: var(--danger); color: #fff; font-size: 13px;
-}
-
-.workspace {
-  flex: 1; display: grid; grid-template-columns: 280px 1fr; gap: 16px;
-  padding: 16px 24px;
-}
-
-.surface { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; }
-
-.run-list { padding: 16px; overflow-y: auto; max-height: calc(100vh - 100px); }
-
-.section-heading {
-  display: flex; justify-content: space-between; align-items: baseline;
-  margin-bottom: 12px; padding-bottom: 8px; border-bottom: 1px solid var(--border);
-}
-
-.section-heading h2 { font-size: 13px; font-weight: 600; }
-.section-heading span { font-size: 11px; color: var(--muted); }
-
-.run-row {
-  width: 100%; display: flex; flex-direction: column; gap: 2px;
-  padding: 10px 12px; margin-bottom: 6px; border: 1px solid var(--border);
-  border-radius: 6px; background: var(--surface); cursor: pointer; text-align: left;
-  font: inherit; color: inherit;
-}
-
-.run-row:hover { border-color: var(--accent); }
-.run-row.selected { border-color: var(--accent); box-shadow: 0 0 0 1px var(--accent); }
-.run-row strong { font-size: 13px; }
-.run-row span { font-size: 11px; color: var(--muted); }
-.run-row small { font-size: 10px; color: var(--muted); }
-
-.content { display: flex; flex-direction: column; gap: 16px; }
-
-.empty {
-  padding: 64px 32px; text-align: center; display: flex; flex-direction: column;
-  align-items: center; gap: 8px; min-height: 400px; justify-content: center;
-}
-.empty h2 { font-size: 16px; font-weight: 600; }
-.empty p { color: var(--muted); }
-
-.summary {
-  display: flex; justify-content: space-between; align-items: center;
-  padding: 16px 20px;
-}
-.summary h2 { font-size: 16px; font-weight: 600; }
-.summary .muted { font-size: 12px; color: var(--muted); margin-top: 2px; }
-
-.review-grid { display: grid; grid-template-columns: 240px 1fr; gap: 16px; }
-
-.task-list { padding: 16px; max-height: calc(100vh - 280px); overflow-y: auto; }
-
-.task-row {
-  width: 100%; display: flex; justify-content: space-between; align-items: center;
-  padding: 8px 10px; margin-bottom: 4px; border: 1px solid var(--border);
-  border-radius: 5px; background: var(--surface); cursor: pointer; text-align: left;
-  font: inherit; color: inherit; font-size: 12px;
-}
-.task-row:hover { border-color: var(--accent); }
-.task-row.selected { border-color: var(--accent); }
-.task-row.removed { opacity: 0.5; }
-.task-row.removed strong { text-decoration: line-through; }
-.task-row span { font-size: 10px; color: var(--muted); }
-
-.task-detail { padding: 20px 24px; max-height: calc(100vh - 280px); overflow-y: auto; }
-
-.detail-header {
-  display: flex; justify-content: space-between; align-items: flex-start;
-  margin-bottom: 20px; padding-bottom: 16px; border-bottom: 1px solid var(--border);
-}
-.detail-header h2 { font-size: 18px; font-weight: 600; }
-
-.detail-block { margin-bottom: 20px; }
-.detail-block h3 {
-  font-size: 12px; font-weight: 600; letter-spacing: 0.04em;
-  text-transform: uppercase; color: var(--muted); margin-bottom: 6px;
-}
-.detail-block pre {
-  padding: 12px 16px; background: var(--code-bg); border-radius: 6px;
-  font-family: var(--mono); font-size: 12px; white-space: pre-wrap;
-  word-break: break-all; max-height: 400px; overflow-y: auto; line-height: 1.6;
-}
-.patch-files {
-  display: grid; gap: 10px; padding: 8px; max-height: 600px; overflow: auto;
-  background: var(--code-bg); border-radius: 6px;
-}
-.patch-files > div { overflow: hidden; border: 1px solid var(--border); border-radius: 4px; }
-
-.button {
-  padding: 7px 14px; border: 1px solid var(--border); border-radius: 6px;
-  background: var(--surface); color: var(--text); font-size: 12px; font-weight: 500;
-  cursor: pointer; font-family: var(--font);
-}
-.button:hover { border-color: var(--accent); color: var(--accent); }
-.button.primary { background: var(--accent); color: #fff; border-color: var(--accent); }
-.button.primary:hover { opacity: 0.9; color: #fff; }
-.button.danger { color: var(--danger); border-color: var(--danger); }
-.button.danger:hover { background: var(--danger); color: #fff; }
-.button.undo { color: var(--success); border-color: var(--success); }
-.button.undo:hover { background: var(--success); color: #fff; }
-
-.muted { color: var(--muted); }

package/review/src/types.ts

@@ -1,47 +0,0 @@
-export interface RunSummary {
-  runId: string;
-  status: string;
-  startedAt: string;
-  closedAt?: string;
-}
-
-export interface TaskProgress {
-  taskId: string;
-  candidateId: string;
-  difficulty: "easy" | "medium" | "hard";
-  status: string;
-  reason?: string;
-}
-
-export interface RunStatus {
-  runId: string;
-  phase: string;
-  requested: number;
-  accepted: number;
-  rejected: number;
-  tasks: TaskProgress[];
-  export?: { uri: string; sizeBytes: number; contentType: string };
-  error?: string;
-}
-
-export interface ExportManifest {
-  schemaVersion: number;
-  runId: string;
-  candidateCounts: Record<string, number>;
-  repository: { url: string; commit: string };
-  version: Record<string, unknown>;
-  acceptedCount: number;
-  tasks: { taskId: string; sha256: string }[];
-}
-
-export interface ExportTask {
-  taskId: string;
-  files: Map<string, Uint8Array>;
-  textFiles: Map<string, string>;
-}
-
-export interface LoadedExport {
-  manifest: ExportManifest;
-  tasks: ExportTask[];
-  taskArchives: Map<string, Uint8Array>;
-}

package/review/tsconfig.json

@@ -1,16 +0,0 @@
-{
-  "compilerOptions": {
-    "allowSyntheticDefaultImports": true,
-    "esModuleInterop": true,
-    "jsx": "react-jsx",
-    "lib": ["DOM", "DOM.Iterable", "ES2023"],
-    "module": "ESNext",
-    "moduleResolution": "Bundler",
-    "noEmit": true,
-    "skipLibCheck": true,
-    "strict": true,
-    "target": "ES2022",
-    "types": ["bun", "vite/client"]
-  },
-  "include": ["src", "vite.config.ts"]
-}

package/review/vite.config.ts

@@ -1,20 +0,0 @@
-import react from "@vitejs/plugin-react";
-import { defineConfig } from "vite";
-
-export default defineConfig({
-  root: "review",
-  plugins: [react()],
-  build: {
-    outDir: "../dist/review",
-    emptyOutDir: true,
-  },
-  server: {
-    allowedHosts: ["avyays-mac-mini.tailf3cee5.ts.net"],
-    proxy: {
-      "/v1": "http://127.0.0.1:8080",
-    },
-  },
-  preview: {
-    allowedHosts: ["avyays-mac-mini.tailf3cee5.ts.net"],
-  },
-});

package/scripts/verify-package.ts

@@ -1,81 +0,0 @@
-import { mkdtemp, readFile, rm } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join, resolve } from "node:path";
-
-const options = process.argv.slice(2);
-const dockerBuild = options.length === 1 && options[0] === "--docker-build";
-if (options.length > 0 && !dockerBuild) {
-  throw new Error(`unknown verify-package argument: ${options.join(" ")}`);
-}
-const root = resolve(import.meta.dir, "..");
-const temporary = await mkdtemp(join(tmpdir(), "self-bench-package-"));
-
-async function run(command: string, args: string[], cwd = root): Promise<string> {
-  const child = Bun.spawn([command, ...args], {
-    cwd,
-    stdout: "pipe",
-    stderr: "pipe",
-  });
-  const [exitCode, stdout, stderr] = await Promise.all([
-    child.exited,
-    new Response(child.stdout).text(),
-    new Response(child.stderr).text(),
-  ]);
-  if (exitCode !== 0) {
-    throw new Error(`${command} ${args.join(" ")} failed:\n${stderr || stdout}`);
-  }
-  return stdout;
-}
-
-try {
-  await run("bun", ["pm", "pack", "--destination", temporary]);
-  const tarballs = (await Array.fromAsync(new Bun.Glob("*.tgz").scan({ cwd: temporary }))).map(
-    (name) => join(temporary, name),
-  );
-  const tarball = tarballs[0] ?? (() => {
-    throw new Error("bun pm pack did not create a tarball");
-  })();
-  const packageJson = JSON.parse(await readFile(join(root, "package.json"), "utf8")) as {
-    name: string;
-    version: string;
-  };
-  const installRoot = join(temporary, "install");
-  await run("mkdir", ["-p", installRoot]);
-  await run("bun", ["init", "--yes"], installRoot);
-  await run("bun", ["add", "--no-save", tarball], installRoot);
-  const executable = join(installRoot, "node_modules", ".bin", "self-bench");
-  const help = await run(executable, ["--help"], installRoot);
-  if (!help.includes("self-bench up")) {
-    throw new Error("installed self-bench did not print the expected CLI help");
-  }
-  const installedRoot = join(installRoot, "node_modules", packageJson.name);
-  for (const asset of [
-    ".dockerignore",
-    "compose.yaml",
-    "Dockerfile",
-    "Dockerfile.sandbox",
-    "review/index.html",
-    "review/src/App.tsx",
-    "review/vite.config.ts",
-    "src/extensions/authoring.ts",
-    "src/skills/selfbench/SKILL.md",
-  ]) {
-    await readFile(join(installedRoot, asset));
-  }
-  const installedPackage = JSON.parse(
-    await readFile(join(installedRoot, "package.json"), "utf8"),
-  ) as { name: string; version: string };
-  if (installedPackage.name !== packageJson.name || installedPackage.version !== packageJson.version) {
-    throw new Error("installed package metadata does not match the workspace package");
-  }
-  if (dockerBuild) {
-    await run(
-      "docker",
-      ["build", "--target", "build", "--file", join(installedRoot, "Dockerfile"), installedRoot],
-      installedRoot,
-    );
-  }
-  console.log(`verified ${installedPackage.name}@${installedPackage.version}`);
-} finally {
-  await rm(temporary, { recursive: true, force: true });
-}

package/scripts/write-build-metadata.ts

@@ -1,27 +0,0 @@
-import { mkdir, writeFile } from "node:fs/promises";
-import { execFile } from "node:child_process";
-import { promisify } from "node:util";
-
-const execFileAsync = promisify(execFile);
-const root = new URL("../", import.meta.url);
-const output = new URL("../dist/build-metadata.js", import.meta.url);
-const configuredCommit = process.env.SELFBENCH_BUILD_COMMIT;
-
-let commit = configuredCommit;
-if (!commit) {
-  try {
-    const result = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root });
-    commit = result.stdout.trim();
-  } catch {
-    commit = "0".repeat(40);
-  }
-}
-if (!/^[0-9a-f]{40}$/i.test(commit)) {
-  throw new Error("SELFBENCH_BUILD_COMMIT must be a full commit SHA");
-}
-
-await mkdir(new URL("../dist/", import.meta.url), { recursive: true });
-await writeFile(
-  output,
-  `// Generated during the build.\nexport const buildCommit = ${JSON.stringify(commit.toLowerCase())};\n`,
-);

package/src/agent-smoke-main.ts

@@ -1,64 +0,0 @@
-#!/usr/bin/env node
-
-import { parseArgs } from "node:util";
-import { smokeAllAdapters } from "./agent-smoke.js";
-import { isHarborEnvironment } from "./providers.js";
-
-const parsed = parseArgs({
-  options: {
-    task: { type: "string" },
-    jobs: { type: "string" },
-    harbor: { type: "string" },
-    environment: { type: "string", default: "modal" },
-    concurrency: { type: "string", default: "4" },
-    help: { type: "boolean", short: "h" },
-  },
-  strict: true,
-});
-if (parsed.values.help) {
-  console.log(`Attempt installation and instantiation of every pinned Harbor adapter.
-
-Usage:
-  self-bench-agent-smoke --task DIRECTORY --jobs DIRECTORY [options]
-
-Options:
-  --harbor PATH              Harbor executable (default: harbor)
-  --environment docker|modal Execution environment (default: modal)
-  --concurrency N            Concurrent adapter checks (default: 4)
-  -h, --help                 Show this help`);
-  process.exit(0);
-}
-const environment = parsed.values.environment ?? "modal";
-if (!isHarborEnvironment(environment)) {
-  throw new Error("--environment must be docker or modal");
-}
-const results = await smokeAllAdapters({
-  taskDirectory: parsed.values.task ?? fail("--task is required"),
-  jobsDirectory: parsed.values.jobs ?? fail("--jobs is required"),
-  environment,
-  concurrency: positiveInteger(parsed.values.concurrency, "--concurrency"),
-  ...(parsed.values.harbor ? { harborPath: parsed.values.harbor } : {}),
-});
-console.log(
-  JSON.stringify(
-    {
-      adapters: results.length,
-      installed: results.filter((result) => result.installed).length,
-      failed: results.filter((result) => !result.installed).length,
-    },
-    null,
-    2,
-  ),
-);
-
-function fail(message: string): never {
-  throw new Error(message);
-}
-
-function positiveInteger(value: string | undefined, label: string): number {
-  const parsed = Number(value);
-  if (!Number.isInteger(parsed) || parsed < 1) {
-    throw new Error(`${label} must be a positive integer`);
-  }
-  return parsed;
-}

package/src/agent-smoke.ts

@@ -1,138 +0,0 @@
-import { mkdir, readFile, writeFile } from "node:fs/promises";
-import { join, resolve } from "node:path";
-import { harborChildEnvironment } from "./harbor-environment.js";
-import { parallelMap } from "./parallel.js";
-import { runCommand } from "./process.js";
-import type { HarborEnvironment } from "./providers.js";
-
-export const HARBOR_AGENT_ADAPTERS = [
-  "oracle",
-  "nop",
-  "acp",
-  "terminus-2",
-  "claude-code",
-  "copilot-cli",
-  "aider",
-  "cline-cli",
-  "codex",
-  "cortex-code",
-  "cursor-cli",
-  "gemini-cli",
-  "antigravity-cli",
-  "antigravity-sdk",
-  "rovodev-cli",
-  "goose",
-  "grok-build",
-  "hermes",
-  "kimi-code",
-  "kimi-cli",
-  "langgraph",
-  "deerflow",
-  "mini-swe-agent",
-  "nemo-agent",
-  "swe-agent",
-  "opencode",
-  "mimo",
-  "openclaw",
-  "openhands",
-  "openhands-sdk",
-  "pi",
-  "qwen-coder",
-  "devin",
-  "trae-agent",
-  "computer-1",
-  "eve",
-  "dspy-rlm",
-  "vibe",
-] as const;
-
-export interface AdapterSmokeOptions {
-  readonly taskDirectory: string;
-  readonly jobsDirectory: string;
-  readonly harborPath?: string;
-  readonly environment?: HarborEnvironment;
-  readonly concurrency?: number;
-}
-
-export interface AdapterSmokeResult {
-  readonly agent: (typeof HARBOR_AGENT_ADAPTERS)[number];
-  readonly installed: boolean;
-  readonly exitCode: number;
-  readonly jobName: string;
-  readonly outputTail: string;
-}
-
-export async function smokeAllAdapters(
-  options: AdapterSmokeOptions,
-): Promise<readonly AdapterSmokeResult[]> {
-  const jobsDirectory = resolve(options.jobsDirectory);
-  const reportsDirectory = join(jobsDirectory, "adapter-smoke");
-  await mkdir(reportsDirectory, { recursive: true });
-  const results = await parallelMap(
-    HARBOR_AGENT_ADAPTERS,
-    options.concurrency ?? 4,
-    async (agent) => {
-      const reportPath = join(reportsDirectory, `${agent}.json`);
-      const existing = await readFile(reportPath, "utf8").catch(() => undefined);
-      if (existing) {
-        return JSON.parse(existing) as AdapterSmokeResult;
-      }
-      const jobName = `install-${agent}-${crypto.randomUUID().slice(0, 8)}`;
-      const result = await runCommand(
-        options.harborPath ?? "harbor",
-        [
-          "run",
-          "--path",
-          resolve(options.taskDirectory),
-          "--agent",
-          agent,
-          "--env",
-          options.environment ?? "modal",
-          "--job-name",
-          jobName,
-          "--jobs-dir",
-          jobsDirectory,
-          "--install-only",
-          "--n-concurrent",
-          "1",
-          "--max-retries",
-          "0",
-          "--delete",
-          "--yes",
-          "--quiet",
-        ],
-        {
-          allowFailure: true,
-          env: harborChildEnvironment(),
-          timeoutMs: 60 * 60 * 1000,
-        },
-      );
-      const summary: AdapterSmokeResult = {
-        agent,
-        installed: result.exitCode === 0,
-        exitCode: result.exitCode,
-        jobName,
-        outputTail: `${result.stdout}\n${result.stderr}`.trim().slice(-2_000),
-      };
-      await writeFile(reportPath, `${JSON.stringify(summary, null, 2)}\n`, { flag: "wx" });
-      return summary;
-    },
-  );
-  await writeFile(
-    join(reportsDirectory, "summary.json"),
-    `${JSON.stringify(
-      {
-        schemaVersion: 1,
-        mode: "install-only",
-        environment: options.environment ?? "modal",
-        adapterCount: results.length,
-        installed: results.filter((result) => result.installed).length,
-        failed: results.filter((result) => !result.installed).length,
-        results,
-      },
-      null,
-      2,
-    )}\n`,
-  );
-  return results;
-}

package/src/api-main.ts

@@ -1,12 +0,0 @@
-import { startApi } from "./api.js";
-import { loadConfig } from "./config.js";
-
-const config = loadConfig();
-const stop = await startApi(config);
-console.log(`SelfBench API listening on http://${config.apiHost}:${config.apiPort}`);
-
-for (const signal of ["SIGINT", "SIGTERM"] as const) {
-  process.once(signal, () => {
-    void stop().finally(() => process.exit(0));
-  });
-}