self-bench 0.3.3 → 0.3.5

package/review/src/styles.css

@@ -0,0 +1,162 @@
+:root {
+  --bg: #f4f5f7;
+  --surface: #ffffff;
+  --border: #d8dce3;
+  --text: #1a1d23;
+  --muted: #6b7280;
+  --accent: #2563eb;
+  --danger: #dc2626;
+  --success: #059669;
+  --code-bg: #f1f3f5;
+  --font: -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
+  --mono: "SF Mono", "Cascadia Code", "Fira Code", monospace;
+}
+
+@media (prefers-color-scheme: dark) {
+  :root {
+    --bg: #0d1117;
+    --surface: #161b22;
+    --border: #30363d;
+    --text: #e6edf3;
+    --muted: #8b949e;
+    --accent: #58a6ff;
+    --danger: #f85149;
+    --success: #3fb950;
+    --code-bg: #1c2128;
+  }
+}
+
+* { margin: 0; padding: 0; box-sizing: border-box; }
+
+body {
+  font-family: var(--font);
+  background: var(--bg);
+  color: var(--text);
+  font-size: 14px;
+  line-height: 1.5;
+}
+
+.app-shell { min-height: 100vh; display: flex; flex-direction: column; }
+
+.topbar {
+  display: flex; justify-content: space-between; align-items: center;
+  padding: 16px 24px; border-bottom: 1px solid var(--border);
+  background: var(--surface);
+}
+
+.eyebrow {
+  font-size: 11px; font-weight: 600; letter-spacing: 0.08em;
+  text-transform: uppercase; color: var(--muted);
+}
+
+.topbar h1 { font-size: 18px; font-weight: 600; margin-top: 2px; }
+.topbar-actions { display: flex; gap: 8px; align-items: center; }
+.token-input {
+  padding: 6px 10px; border: 1px solid var(--border); border-radius: 6px;
+  background: var(--surface); color: var(--text); font-size: 12px; width: 180px;
+}
+
+.error-banner {
+  padding: 12px 24px; background: var(--danger); color: #fff; font-size: 13px;
+}
+
+.workspace {
+  flex: 1; display: grid; grid-template-columns: 280px 1fr; gap: 16px;
+  padding: 16px 24px;
+}
+
+.surface { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; }
+
+.run-list { padding: 16px; overflow-y: auto; max-height: calc(100vh - 100px); }
+
+.section-heading {
+  display: flex; justify-content: space-between; align-items: baseline;
+  margin-bottom: 12px; padding-bottom: 8px; border-bottom: 1px solid var(--border);
+}
+
+.section-heading h2 { font-size: 13px; font-weight: 600; }
+.section-heading span { font-size: 11px; color: var(--muted); }
+
+.run-row {
+  width: 100%; display: flex; flex-direction: column; gap: 2px;
+  padding: 10px 12px; margin-bottom: 6px; border: 1px solid var(--border);
+  border-radius: 6px; background: var(--surface); cursor: pointer; text-align: left;
+  font: inherit; color: inherit;
+}
+
+.run-row:hover { border-color: var(--accent); }
+.run-row.selected { border-color: var(--accent); box-shadow: 0 0 0 1px var(--accent); }
+.run-row strong { font-size: 13px; }
+.run-row span { font-size: 11px; color: var(--muted); }
+.run-row small { font-size: 10px; color: var(--muted); }
+
+.content { display: flex; flex-direction: column; gap: 16px; }
+
+.empty {
+  padding: 64px 32px; text-align: center; display: flex; flex-direction: column;
+  align-items: center; gap: 8px; min-height: 400px; justify-content: center;
+}
+.empty h2 { font-size: 16px; font-weight: 600; }
+.empty p { color: var(--muted); }
+
+.summary {
+  display: flex; justify-content: space-between; align-items: center;
+  padding: 16px 20px;
+}
+.summary h2 { font-size: 16px; font-weight: 600; }
+.summary .muted { font-size: 12px; color: var(--muted); margin-top: 2px; }
+
+.review-grid { display: grid; grid-template-columns: 240px 1fr; gap: 16px; }
+
+.task-list { padding: 16px; max-height: calc(100vh - 280px); overflow-y: auto; }
+
+.task-row {
+  width: 100%; display: flex; justify-content: space-between; align-items: center;
+  padding: 8px 10px; margin-bottom: 4px; border: 1px solid var(--border);
+  border-radius: 5px; background: var(--surface); cursor: pointer; text-align: left;
+  font: inherit; color: inherit; font-size: 12px;
+}
+.task-row:hover { border-color: var(--accent); }
+.task-row.selected { border-color: var(--accent); }
+.task-row.removed { opacity: 0.5; }
+.task-row.removed strong { text-decoration: line-through; }
+.task-row span { font-size: 10px; color: var(--muted); }
+
+.task-detail { padding: 20px 24px; max-height: calc(100vh - 280px); overflow-y: auto; }
+
+.detail-header {
+  display: flex; justify-content: space-between; align-items: flex-start;
+  margin-bottom: 20px; padding-bottom: 16px; border-bottom: 1px solid var(--border);
+}
+.detail-header h2 { font-size: 18px; font-weight: 600; }
+
+.detail-block { margin-bottom: 20px; }
+.detail-block h3 {
+  font-size: 12px; font-weight: 600; letter-spacing: 0.04em;
+  text-transform: uppercase; color: var(--muted); margin-bottom: 6px;
+}
+.detail-block pre {
+  padding: 12px 16px; background: var(--code-bg); border-radius: 6px;
+  font-family: var(--mono); font-size: 12px; white-space: pre-wrap;
+  word-break: break-all; max-height: 400px; overflow-y: auto; line-height: 1.6;
+}
+.patch-files {
+  display: grid; gap: 10px; padding: 8px; max-height: 600px; overflow: auto;
+  background: var(--code-bg); border-radius: 6px;
+}
+.patch-files > div { overflow: hidden; border: 1px solid var(--border); border-radius: 4px; }
+
+.button {
+  padding: 7px 14px; border: 1px solid var(--border); border-radius: 6px;
+  background: var(--surface); color: var(--text); font-size: 12px; font-weight: 500;
+  cursor: pointer; font-family: var(--font);
+}
+.button:hover { border-color: var(--accent); color: var(--accent); }
+.button.primary { background: var(--accent); color: #fff; border-color: var(--accent); }
+.button.primary:hover { opacity: 0.9; color: #fff; }
+.button.danger { color: var(--danger); border-color: var(--danger); }
+.button.danger:hover { background: var(--danger); color: #fff; }
+.button.undo { color: var(--success); border-color: var(--success); }
+.button.undo:hover { background: var(--success); color: #fff; }
+
+.muted { color: var(--muted); }

package/review/src/types.ts

@@ -0,0 +1,47 @@
+export interface RunSummary {
+  runId: string;
+  status: string;
+  startedAt: string;
+  closedAt?: string;
+}
+
+export interface TaskProgress {
+  taskId: string;
+  candidateId: string;
+  difficulty: "easy" | "medium" | "hard";
+  status: string;
+  reason?: string;
+}
+
+export interface RunStatus {
+  runId: string;
+  phase: string;
+  requested: number;
+  accepted: number;
+  rejected: number;
+  tasks: TaskProgress[];
+  export?: { uri: string; sizeBytes: number; contentType: string };
+  error?: string;
+}
+
+export interface ExportManifest {
+  schemaVersion: number;
+  runId: string;
+  candidateCounts: Record<string, number>;
+  repository: { url: string; commit: string };
+  version: Record<string, unknown>;
+  acceptedCount: number;
+  tasks: { taskId: string; sha256: string }[];
+}
+
+export interface ExportTask {
+  taskId: string;
+  files: Map<string, Uint8Array>;
+  textFiles: Map<string, string>;
+}
+
+export interface LoadedExport {
+  manifest: ExportManifest;
+  tasks: ExportTask[];
+  taskArchives: Map<string, Uint8Array>;
+}

package/review/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "allowSyntheticDefaultImports": true,
+    "esModuleInterop": true,
+    "jsx": "react-jsx",
+    "lib": ["DOM", "DOM.Iterable", "ES2023"],
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "noEmit": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "target": "ES2022",
+    "types": ["bun", "vite/client"]
+  },
+  "include": ["src", "vite.config.ts"]
+}

package/review/vite.config.ts

@@ -0,0 +1,20 @@
+import react from "@vitejs/plugin-react";
+import { defineConfig } from "vite";
+
+export default defineConfig({
+  root: "review",
+  plugins: [react()],
+  build: {
+    outDir: "../dist/review",
+    emptyOutDir: true,
+  },
+  server: {
+    allowedHosts: ["avyays-mac-mini.tailf3cee5.ts.net"],
+    proxy: {
+      "/v1": "http://127.0.0.1:8080",
+    },
+  },
+  preview: {
+    allowedHosts: ["avyays-mac-mini.tailf3cee5.ts.net"],
+  },
+});

package/scripts/verify-package.ts

@@ -2,8 +2,14 @@ import { mkdtemp, readFile, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 
+const options = process.argv.slice(2);
+const dockerBuild = options.length === 1 && options[0] === "--docker-build";
+if (options.length > 0 && !dockerBuild) {
+  throw new Error(`unknown verify-package argument: ${options.join(" ")}`);
+}
 const root = resolve(import.meta.dir, "..");
 const temporary = await mkdtemp(join(tmpdir(), "self-bench-package-"));
+
 async function run(command: string, args: string[], cwd = root): Promise<string> {
   const child = Bun.spawn([command, ...args], {
     cwd,
@@ -42,15 +48,33 @@ try {
   if (!help.includes("self-bench up")) {
     throw new Error("installed self-bench did not print the expected CLI help");
   }
-  for (const asset of ["compose.yaml", "Dockerfile", "Dockerfile.sandbox", "src/skills/selfbench/SKILL.md"]) {
-    await readFile(join(installRoot, "node_modules", packageJson.name, asset));
+  const installedRoot = join(installRoot, "node_modules", packageJson.name);
+  for (const asset of [
+    ".dockerignore",
+    "compose.yaml",
+    "Dockerfile",
+    "Dockerfile.sandbox",
+    "review/index.html",
+    "review/src/App.tsx",
+    "review/vite.config.ts",
+    "src/extensions/authoring.ts",
+    "src/skills/selfbench/SKILL.md",
+  ]) {
+    await readFile(join(installedRoot, asset));
   }
   const installedPackage = JSON.parse(
-    await readFile(join(installRoot, "node_modules", packageJson.name, "package.json"), "utf8"),
+    await readFile(join(installedRoot, "package.json"), "utf8"),
   ) as { name: string; version: string };
   if (installedPackage.name !== packageJson.name || installedPackage.version !== packageJson.version) {
     throw new Error("installed package metadata does not match the workspace package");
   }
+  if (dockerBuild) {
+    await run(
+      "docker",
+      ["build", "--target", "build", "--file", join(installedRoot, "Dockerfile"), installedRoot],
+      installedRoot,
+    );
+  }
   console.log(`verified ${installedPackage.name}@${installedPackage.version}`);
 } finally {
   await rm(temporary, { recursive: true, force: true });

package/src/agent-smoke-main.ts

@@ -2,6 +2,7 @@
 
 import { parseArgs } from "node:util";
 import { smokeAllAdapters } from "./agent-smoke.js";
+import { isHarborEnvironment } from "./providers.js";
 
 const parsed = parseArgs({
   options: {
@@ -27,8 +28,8 @@ Options:
   -h, --help                 Show this help`);
   process.exit(0);
 }
-const environment = parsed.values.environment;
-if (environment !== "docker" && environment !== "modal") {
+const environment = parsed.values.environment ?? "modal";
+if (!isHarborEnvironment(environment)) {
   throw new Error("--environment must be docker or modal");
 }
 const results = await smokeAllAdapters({

package/src/agent-smoke.ts

@@ -1,7 +1,9 @@
 import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { join, resolve } from "node:path";
+import { harborChildEnvironment } from "./harbor-environment.js";
 import { parallelMap } from "./parallel.js";
 import { runCommand } from "./process.js";
+import type { HarborEnvironment } from "./providers.js";
 
 export const HARBOR_AGENT_ADAPTERS = [
   "oracle",
@@ -48,7 +50,7 @@ export interface AdapterSmokeOptions {
   readonly taskDirectory: string;
   readonly jobsDirectory: string;
   readonly harborPath?: string;
-  readonly environment?: "docker" | "modal";
+  readonly environment?: HarborEnvironment;
   readonly concurrency?: number;
 }
 
@@ -99,7 +101,11 @@ export async function smokeAllAdapters(
           "--yes",
           "--quiet",
         ],
-        { allowFailure: true, timeoutMs: 60 * 60 * 1000 },
+        {
+          allowFailure: true,
+          env: harborChildEnvironment(),
+          timeoutMs: 60 * 60 * 1000,
+        },
       );
       const summary: AdapterSmokeResult = {
         agent,

package/src/api.ts

@@ -11,6 +11,7 @@ import type { SelfBenchConfig } from "./config.js";
 import {
   artifactRefSchema,
   type RunPhase,
+  type RunRequest,
   type RunStatus,
   repositoryRefSchema,
   runRequestSchema,
@@ -68,26 +69,10 @@ export async function startApi(config: SelfBenchConfig): Promise<() => Promise<v
         return;
       }
       if (request.method === "POST" && url.pathname === "/v1/runs") {
-        const submission = submissionSchema.parse(
+        const workflowInput = buildRunRequest(
+          config,
           JSON.parse((await readBody(request)).toString("utf8")),
         );
-        const workflowInput = runRequestSchema.parse({
-          runId: submission.runId,
-          repository: submission.repository,
-          provenance: submission.provenance,
-          candidateCounts: submission.candidateCounts,
-          authoring: {
-            provider: "openai-codex",
-            model: submission.authoringModel,
-            reasoningEffort: "high",
-          },
-          version: {
-            selfbenchCommit: config.buildCommit ?? submission.selfbenchCommit,
-            executionBackend: config.execution.kind,
-            sandboxImage: config.execution.image,
-            schema: 1,
-          },
-        });
         await client.workflow.start(selfBenchRunWorkflow, {
           workflowId: workflowInput.runId,
           taskQueue: config.temporal.taskQueue,
@@ -167,6 +152,34 @@ export async function startApi(config: SelfBenchConfig): Promise<() => Promise<v
   };
 }
 
+export function buildRunRequest(
+  config: SelfBenchConfig,
+  submission: z.input<typeof submissionSchema>,
+): RunRequest {
+  const parsed = submissionSchema.parse(submission);
+  return runRequestSchema.parse({
+    runId: parsed.runId,
+    repository: parsed.repository,
+    provenance: parsed.provenance,
+    candidateCounts: parsed.candidateCounts,
+    authoring: {
+      provider: "openai-codex",
+      model: parsed.authoringModel,
+      reasoningEffort: "high",
+    },
+    version: {
+      selfbenchCommit: config.buildCommit ?? parsed.selfbenchCommit,
+      executionBackend: config.execution.kind,
+      harborEnvironment: config.harborEnvironment,
+      sandboxImage: config.execution.image,
+      ...(config.execution.kind === "vercel"
+        ? { sandboxTimeoutCapMs: config.execution.timeoutCapMs }
+        : {}),
+      schema: 1,
+    },
+  });
+}
+
 async function queryStatus(
   handle: ReturnType<Client["workflow"]["getHandle"]>,
 ): Promise<RunStatus | object> {

package/src/cli.ts

@@ -10,12 +10,20 @@ import type { ReadableStream as NodeReadableStream } from "node:stream/web";
 import { fileURLToPath } from "node:url";
 import { parseArgs } from "node:util";
 import { buildCommit } from "./build-metadata.js";
+import { loadWorkerConfig } from "./config.js";
 import { runCommand } from "./process.js";
 import { collectGitHubPullRequestProvenance, collectRepositoryProvenance } from "./provenance.js";
+import { isExecutionBackend, isHarborEnvironment, matchingHarborEnvironment } from "./providers.js";
 import { type PolledRunStatus, waitForRun } from "./run-wait.js";
+import { SetupCanceledError } from "./terminal-prompts.js";
+import { applyVercelProfile } from "./vercel-profile.js";
+import { setupVercel } from "./vercel-setup.js";
 
 const [command, ...rest] = process.argv.slice(2);
 switch (command) {
+  case "setup":
+    await setup(rest);
+    break;
   case "up":
     await up(rest);
     break;
@@ -47,31 +55,74 @@ switch (command) {
     throw new Error(`unknown command: ${command}`);
 }
 
+async function setup(args: string[]): Promise<void> {
+  const [provider, ...providerArgs] = args;
+  if (provider !== "vercel") {
+    fail("setup currently supports only: self-bench setup vercel");
+  }
+  const parsed = parseArgs({
+    args: providerArgs,
+    options: {
+      profile: { type: "string", default: "default" },
+      verbose: { type: "boolean", default: false },
+    },
+    strict: true,
+  });
+  try {
+    await setupVercel({
+      profileName: parsed.values.profile,
+      verbose: parsed.values.verbose,
+    });
+  } catch (error) {
+    if (error instanceof SetupCanceledError) {
+      process.exitCode = 130;
+      return;
+    }
+    throw error;
+  }
+}
+
 async function up(args: string[]): Promise<void> {
   const parsed = parseArgs({
     args,
     options: {
       backend: { type: "string", default: "docker" },
+      "harbor-environment": { type: "string" },
       "modal-config": { type: "string" },
+      "vercel-profile": { type: "string" },
     },
     strict: true,
   });
-  if (parsed.values.backend !== "docker" && parsed.values.backend !== "modal") {
-    fail('--backend must be "docker" or "modal"');
+  if (!isExecutionBackend(parsed.values.backend)) {
+    fail('--backend must be "docker", "modal", or "vercel"');
   }
-  if (parsed.values.backend === "docker" && parsed.values["modal-config"] !== undefined) {
-    fail("--modal-config requires --backend modal");
+  if (
+    parsed.values["harbor-environment"] !== undefined &&
+    !isHarborEnvironment(parsed.values["harbor-environment"])
+  ) {
+    fail('--harbor-environment must be "docker" or "modal"');
   }
 
   const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
   const composeFile = resolve(projectRoot, "compose.yaml");
   const backend = parsed.values.backend;
-  const environment = {
+  const harborEnvironment =
+    parsed.values["harbor-environment"] ??
+    matchingHarborEnvironment(backend) ??
+    fail(`--harbor-environment is required with --backend ${backend}`);
+  const usesModal = backend === "modal" || harborEnvironment === "modal";
+  if (!usesModal && parsed.values["modal-config"] !== undefined) {
+    fail("--modal-config requires Modal generation or Harbor");
+  }
+  if (backend !== "vercel" && parsed.values["vercel-profile"] !== undefined) {
+    fail("--vercel-profile requires Vercel generation");
+  }
+  let environment: NodeJS.ProcessEnv = {
     ...process.env,
     SELFBENCH_BUILD_COMMIT: await resolveSelfBenchCommit(),
     SELFBENCH_EXECUTION_BACKEND: backend,
-    SELFBENCH_HARBOR_ENVIRONMENT: backend,
-    ...(backend === "modal"
+    SELFBENCH_HARBOR_ENVIRONMENT: harborEnvironment,
+    ...(usesModal
       ? {
           SELFBENCH_MODAL_CONFIG_PATH: resolve(
             parsed.values["modal-config"] ?? resolve(homedir(), ".modal.toml"),
@@ -79,6 +130,10 @@ async function up(args: string[]): Promise<void> {
         }
       : {}),
   };
+  if (backend === "vercel") {
+    environment = await applyVercelProfile(environment, parsed.values["vercel-profile"]);
+    loadWorkerConfig(environment);
+  }
 
   if (backend === "docker") {
     await runCommand(
@@ -97,7 +152,9 @@ async function up(args: string[]): Promise<void> {
   await runCommand("docker", ["compose", "--file", composeFile, "up", "-d", "--build"], {
     env: environment,
   });
-  console.log(`SelfBench is running with the ${backend} backend at http://127.0.0.1:8080`);
+  console.log(
+    `SelfBench is running with ${backend} generation and ${harborEnvironment} Harbor at http://127.0.0.1:8080`,
+  );
 }
 
 async function down(): Promise<void> {
@@ -340,7 +397,9 @@ function printHelp(): void {
   console.log(`SelfBench creates durable tiered Harbor evaluations.
 
 Usage:
-  self-bench up [--backend docker|modal] [--modal-config PATH]
+  self-bench setup vercel [--profile NAME] [--verbose]
+  self-bench up [--backend docker|modal|vercel] [--harbor-environment docker|modal]
+                [--modal-config PATH] [--vercel-profile NAME]
   self-bench down
   self-bench run --repo PATH [--easy-count N] [--medium-count N] [--hard-count N]
                   [--model MODEL] [--run-id ID] [--wait] [--output OUTPUT.tar.gz]
@@ -349,8 +408,10 @@ Usage:
   self-bench download RUN_ID OUTPUT.tar.gz
   self-bench list
 
-The up command starts the local stack and configures both sandbox execution and Harbor validation for
-one backend. Modal uses ~/.modal.toml unless --modal-config overrides it.
+The up command starts the local stack. Docker and Modal default Harbor to the matching backend; Vercel
+requires --harbor-environment because Harbor does not support Vercel. Modal generation or Harbor uses
+~/.modal.toml unless --modal-config overrides it. Run self-bench setup vercel once to create or select a
+project, publish the pinned runtime image, verify access, and save an owner-only local profile.
 
 The tier counts are candidate authoring budgets, not accepted-task targets. Rejected candidates are not
 replaced, and the export contains only accepted tasks. The run command performs only repository metadata