self-bench 0.3.0 → 0.3.2

package/src/agent-smoke-main.ts

@@ -0,0 +1,63 @@
+#!/usr/bin/env node
+
+import { parseArgs } from "node:util";
+import { smokeAllAdapters } from "./agent-smoke.js";
+
+const parsed = parseArgs({
+  options: {
+    task: { type: "string" },
+    jobs: { type: "string" },
+    harbor: { type: "string" },
+    environment: { type: "string", default: "modal" },
+    concurrency: { type: "string", default: "4" },
+    help: { type: "boolean", short: "h" },
+  },
+  strict: true,
+});
+if (parsed.values.help) {
+  console.log(`Attempt installation and instantiation of every pinned Harbor adapter.
+
+Usage:
+  self-bench-agent-smoke --task DIRECTORY --jobs DIRECTORY [options]
+
+Options:
+  --harbor PATH              Harbor executable (default: harbor)
+  --environment docker|modal Execution environment (default: modal)
+  --concurrency N            Concurrent adapter checks (default: 4)
+  -h, --help                 Show this help`);
+  process.exit(0);
+}
+const environment = parsed.values.environment;
+if (environment !== "docker" && environment !== "modal") {
+  throw new Error("--environment must be docker or modal");
+}
+const results = await smokeAllAdapters({
+  taskDirectory: parsed.values.task ?? fail("--task is required"),
+  jobsDirectory: parsed.values.jobs ?? fail("--jobs is required"),
+  environment,
+  concurrency: positiveInteger(parsed.values.concurrency, "--concurrency"),
+  ...(parsed.values.harbor ? { harborPath: parsed.values.harbor } : {}),
+});
+console.log(
+  JSON.stringify(
+    {
+      adapters: results.length,
+      installed: results.filter((result) => result.installed).length,
+      failed: results.filter((result) => !result.installed).length,
+    },
+    null,
+    2,
+  ),
+);
+
+function fail(message: string): never {
+  throw new Error(message);
+}
+
+function positiveInteger(value: string | undefined, label: string): number {
+  const parsed = Number(value);
+  if (!Number.isInteger(parsed) || parsed < 1) {
+    throw new Error(`${label} must be a positive integer`);
+  }
+  return parsed;
+}

package/src/agent-smoke.ts

@@ -0,0 +1,132 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { parallelMap } from "./parallel.js";
+import { runCommand } from "./process.js";
+
+export const HARBOR_AGENT_ADAPTERS = [
+  "oracle",
+  "nop",
+  "acp",
+  "terminus-2",
+  "claude-code",
+  "copilot-cli",
+  "aider",
+  "cline-cli",
+  "codex",
+  "cortex-code",
+  "cursor-cli",
+  "gemini-cli",
+  "antigravity-cli",
+  "antigravity-sdk",
+  "rovodev-cli",
+  "goose",
+  "grok-build",
+  "hermes",
+  "kimi-code",
+  "kimi-cli",
+  "langgraph",
+  "deerflow",
+  "mini-swe-agent",
+  "nemo-agent",
+  "swe-agent",
+  "opencode",
+  "mimo",
+  "openclaw",
+  "openhands",
+  "openhands-sdk",
+  "pi",
+  "qwen-coder",
+  "devin",
+  "trae-agent",
+  "computer-1",
+  "eve",
+  "dspy-rlm",
+  "vibe",
+] as const;
+
+export interface AdapterSmokeOptions {
+  readonly taskDirectory: string;
+  readonly jobsDirectory: string;
+  readonly harborPath?: string;
+  readonly environment?: "docker" | "modal";
+  readonly concurrency?: number;
+}
+
+export interface AdapterSmokeResult {
+  readonly agent: (typeof HARBOR_AGENT_ADAPTERS)[number];
+  readonly installed: boolean;
+  readonly exitCode: number;
+  readonly jobName: string;
+  readonly outputTail: string;
+}
+
+export async function smokeAllAdapters(
+  options: AdapterSmokeOptions,
+): Promise<readonly AdapterSmokeResult[]> {
+  const jobsDirectory = resolve(options.jobsDirectory);
+  const reportsDirectory = join(jobsDirectory, "adapter-smoke");
+  await mkdir(reportsDirectory, { recursive: true });
+  const results = await parallelMap(
+    HARBOR_AGENT_ADAPTERS,
+    options.concurrency ?? 4,
+    async (agent) => {
+      const reportPath = join(reportsDirectory, `${agent}.json`);
+      const existing = await readFile(reportPath, "utf8").catch(() => undefined);
+      if (existing) {
+        return JSON.parse(existing) as AdapterSmokeResult;
+      }
+      const jobName = `install-${agent}-${crypto.randomUUID().slice(0, 8)}`;
+      const result = await runCommand(
+        options.harborPath ?? "harbor",
+        [
+          "run",
+          "--path",
+          resolve(options.taskDirectory),
+          "--agent",
+          agent,
+          "--env",
+          options.environment ?? "modal",
+          "--job-name",
+          jobName,
+          "--jobs-dir",
+          jobsDirectory,
+          "--install-only",
+          "--n-concurrent",
+          "1",
+          "--max-retries",
+          "0",
+          "--delete",
+          "--yes",
+          "--quiet",
+        ],
+        { allowFailure: true, timeoutMs: 60 * 60 * 1000 },
+      );
+      const summary: AdapterSmokeResult = {
+        agent,
+        installed: result.exitCode === 0,
+        exitCode: result.exitCode,
+        jobName,
+        outputTail: `${result.stdout}\n${result.stderr}`.trim().slice(-2_000),
+      };
+      await writeFile(reportPath, `${JSON.stringify(summary, null, 2)}\n`, { flag: "wx" });
+      return summary;
+    },
+  );
+  await writeFile(
+    join(reportsDirectory, "summary.json"),
+    `${JSON.stringify(
+      {
+        schemaVersion: 1,
+        mode: "install-only",
+        environment: options.environment ?? "modal",
+        adapterCount: results.length,
+        installed: results.filter((result) => result.installed).length,
+        failed: results.filter((result) => !result.installed).length,
+        results,
+      },
+      null,
+      2,
+    )}\n`,
+  );
+  return results;
+}

package/src/api-main.ts

@@ -0,0 +1,12 @@
+import { startApi } from "./api.js";
+import { loadConfig } from "./config.js";
+
+const config = loadConfig();
+const stop = await startApi(config);
+console.log(`SelfBench API listening on http://${config.apiHost}:${config.apiPort}`);
+
+for (const signal of ["SIGINT", "SIGTERM"] as const) {
+  process.once(signal, () => {
+    void stop().finally(() => process.exit(0));
+  });
+}

package/src/api.ts

@@ -0,0 +1,239 @@
+import { timingSafeEqual } from "node:crypto";
+import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
+import { pipeline } from "node:stream/promises";
+import { Client } from "@temporalio/client";
+import { z } from "zod";
+import { createArtifactStore } from "./artifacts.js";
+import type { SelfBenchConfig } from "./config.js";
+import {
+  artifactRefSchema,
+  type RunPhase,
+  type RunStatus,
+  repositoryRefSchema,
+  runRequestSchema,
+} from "./contracts.js";
+import { connectTemporalClient } from "./temporal.js";
+import { selfBenchRunWorkflow, statusQuery } from "./workflow.js";
+
+const submissionSchema = z.object({
+  runId: z.string().regex(/^[a-z0-9][a-z0-9-]{2,62}$/),
+  repository: repositoryRefSchema,
+  provenance: artifactRefSchema,
+  candidateCounts: z.object({
+    easy: z.number().int().min(0).max(100),
+    medium: z.number().int().min(0).max(100),
+    hard: z.number().int().min(0).max(100),
+  }),
+  authoringModel: z.string().min(1).default("gpt-5.6-sol"),
+  selfbenchCommit: z.string().regex(/^[0-9a-f]{40}$/i),
+});
+
+export async function startApi(config: SelfBenchConfig): Promise<() => Promise<void>> {
+  const connection = await connectTemporalClient(config.temporal);
+  const client = new Client({ connection, namespace: config.temporal.namespace });
+  const artifacts = createArtifactStore(config.artifact);
+  const server = createServer(async (request, response) => {
+    try {
+      const url = new URL(request.url ?? "/", `http://${request.headers.host ?? "localhost"}`);
+      if (request.method === "GET" && url.pathname === "/healthz") {
+        sendJson(response, 200, { ok: true });
+        return;
+      }
+      if (!authorized(request, config.apiToken)) {
+        sendJson(response, 401, { error: "unauthorized" });
+        return;
+      }
+      if (request.method === "POST" && url.pathname === "/v1/provenance") {
+        const runId = z
+          .string()
+          .regex(/^[a-z0-9][a-z0-9-]{2,62}$/)
+          .parse(url.searchParams.get("runId"));
+        const body = await readBody(request, 100 * 1024 * 1024);
+        const reference = await artifacts.put(
+          `runs/${runId}/input/provenance.jsonl`,
+          body,
+          "application/x-ndjson",
+        );
+        sendJson(response, 201, reference);
+        return;
+      }
+      if (request.method === "POST" && url.pathname === "/v1/runs") {
+        const submission = submissionSchema.parse(
+          JSON.parse((await readBody(request)).toString("utf8")),
+        );
+        const workflowInput = runRequestSchema.parse({
+          runId: submission.runId,
+          repository: submission.repository,
+          provenance: submission.provenance,
+          candidateCounts: submission.candidateCounts,
+          authoring: {
+            provider: "openai-codex",
+            model: submission.authoringModel,
+            reasoningEffort: "high",
+          },
+          version: {
+            selfbenchCommit: config.buildCommit ?? submission.selfbenchCommit,
+            executionBackend: config.execution.kind,
+            sandboxImage: config.execution.image,
+            schema: 1,
+          },
+        });
+        await client.workflow.start(selfBenchRunWorkflow, {
+          workflowId: workflowInput.runId,
+          taskQueue: config.temporal.taskQueue,
+          args: [workflowInput],
+          workflowExecutionTimeout: "14 days",
+        });
+        sendJson(response, 202, { runId: workflowInput.runId });
+        return;
+      }
+      const runMatch = /^\/v1\/runs\/([a-z0-9][a-z0-9-]{2,62})(?:\/(cancel|export))?$/.exec(
+        url.pathname,
+      );
+      if (runMatch?.[1] && request.method === "GET" && runMatch[2] === "export") {
+        const status = await queryStatus(client.workflow.getHandle(runMatch[1]));
+        if (!("export" in status) || !status.export) {
+          sendJson(response, 409, { error: "run export is not ready" });
+          return;
+        }
+        const body = await artifacts.openRead(status.export);
+        response.writeHead(200, {
+          "content-type": status.export.contentType,
+          "content-length": status.export.sizeBytes,
+          "content-disposition": `attachment; filename="selfbench-${runMatch[1]}.tar.gz"`,
+          "x-content-sha256": status.export.sha256,
+        });
+        await pipeline(body, response);
+        return;
+      }
+      if (runMatch?.[1] && request.method === "GET" && !runMatch[2]) {
+        const handle = client.workflow.getHandle(runMatch[1]);
+        const status = await queryStatus(handle);
+        sendJson(response, 200, status);
+        return;
+      }
+      if (runMatch?.[1] && request.method === "POST" && runMatch[2] === "cancel") {
+        await client.workflow.getHandle(runMatch[1]).cancel();
+        sendJson(response, 202, { runId: runMatch[1], cancellationRequested: true });
+        return;
+      }
+      if (request.method === "GET" && url.pathname === "/v1/runs") {
+        const runs: unknown[] = [];
+        for await (const execution of client.workflow.list({
+          query: "WorkflowType = 'selfBenchRunWorkflow'",
+        })) {
+          runs.push({
+            runId: execution.workflowId,
+            status: execution.status.name,
+            startedAt: execution.startTime.toISOString(),
+            closedAt: execution.closeTime?.toISOString(),
+          });
+        }
+        sendJson(response, 200, runs);
+        return;
+      }
+      sendJson(response, 404, { error: "not found" });
+    } catch (error) {
+      if (response.headersSent) {
+        response.destroy(error instanceof Error ? error : new Error(String(error)));
+        return;
+      }
+      const message = error instanceof Error ? error.message : String(error);
+      sendJson(response, error instanceof z.ZodError || error instanceof SyntaxError ? 400 : 500, {
+        error: message,
+      });
+    }
+  });
+
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(config.apiPort, config.apiHost, resolve);
+  });
+  return async () => {
+    await new Promise<void>((resolve, reject) =>
+      server.close((error) => (error ? reject(error) : resolve())),
+    );
+    await connection.close();
+  };
+}
+
+async function queryStatus(
+  handle: ReturnType<Client["workflow"]["getHandle"]>,
+): Promise<RunStatus | object> {
+  try {
+    const [status, description] = await Promise.all([handle.query(statusQuery), handle.describe()]);
+    if (description.status.name === "RUNNING" || terminalRunPhase(status.phase)) {
+      return status;
+    }
+    const phase = executionPhase(description.status.name);
+    return {
+      ...status,
+      phase,
+      ...(phase === "failed" && !status.error
+        ? { error: `Temporal workflow ${description.status.name.toLowerCase()}` }
+        : {}),
+    };
+  } catch {
+    const description = await handle.describe();
+    return {
+      runId: description.workflowId,
+      phase: executionPhase(description.status.name),
+    };
+  }
+}
+
+function terminalRunPhase(phase: RunPhase): boolean {
+  return ["complete", "blocked", "failed", "cancelled"].includes(phase);
+}
+
+function executionPhase(status: string): RunPhase {
+  switch (status) {
+    case "COMPLETED":
+      return "complete";
+    case "CANCELED":
+      return "cancelled";
+    case "RUNNING":
+      return "queued";
+    default:
+      return "failed";
+  }
+}
+
+async function readBody(request: IncomingMessage, limit = 10 * 1024 * 1024): Promise<Buffer> {
+  const chunks: Buffer[] = [];
+  let size = 0;
+  for await (const chunk of request) {
+    const value = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+    size += value.byteLength;
+    if (size > limit) {
+      throw new Error(`request body exceeds ${limit} bytes`);
+    }
+    chunks.push(value);
+  }
+  return Buffer.concat(chunks);
+}
+
+function authorized(request: IncomingMessage, token: string | undefined): boolean {
+  if (!token) {
+    return true;
+  }
+  const supplied = request.headers.authorization?.replace(/^Bearer\s+/i, "");
+  if (!supplied) {
+    return false;
+  }
+  const expectedBuffer = Buffer.from(token);
+  const suppliedBuffer = Buffer.from(supplied);
+  return (
+    expectedBuffer.length === suppliedBuffer.length &&
+    timingSafeEqual(expectedBuffer, suppliedBuffer)
+  );
+}
+
+function sendJson(response: ServerResponse, status: number, value: unknown): void {
+  const body = `${JSON.stringify(value, null, 2)}\n`;
+  response.writeHead(status, {
+    "content-type": "application/json; charset=utf-8",
+    "content-length": Buffer.byteLength(body),
+  });
+  response.end(body);
+}