self-bench 0.3.0 → 0.3.3

package/src/api.ts

@@ -0,0 +1,277 @@
+import { timingSafeEqual } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
+import { dirname, join } from "node:path";
+import { pipeline } from "node:stream/promises";
+import { fileURLToPath } from "node:url";
+import { Client } from "@temporalio/client";
+import { z } from "zod";
+import { createArtifactStore } from "./artifacts.js";
+import type { SelfBenchConfig } from "./config.js";
+import {
+  artifactRefSchema,
+  type RunPhase,
+  type RunStatus,
+  repositoryRefSchema,
+  runRequestSchema,
+} from "./contracts.js";
+import { connectTemporalClient } from "./temporal/connection.js";
+import { selfBenchRunWorkflow, statusQuery } from "./temporal/workflow.js";
+
+const submissionSchema = z.object({
+  runId: z.string().regex(/^[a-z0-9][a-z0-9-]{2,62}$/),
+  repository: repositoryRefSchema,
+  provenance: artifactRefSchema,
+  candidateCounts: z.object({
+    easy: z.number().int().min(0).max(100),
+    medium: z.number().int().min(0).max(100),
+    hard: z.number().int().min(0).max(100),
+  }),
+  authoringModel: z.string().min(1).default("gpt-5.6-sol"),
+  selfbenchCommit: z.string().regex(/^[0-9a-f]{40}$/i),
+});
+
+export async function startApi(config: SelfBenchConfig): Promise<() => Promise<void>> {
+  const connection = await connectTemporalClient(config.temporal);
+  const client = new Client({ connection, namespace: config.temporal.namespace });
+  const artifacts = createArtifactStore(config.artifact);
+  const server = createServer(async (request, response) => {
+    try {
+      const url = new URL(request.url ?? "/", `http://${request.headers.host ?? "localhost"}`);
+      if (request.method === "GET" && url.pathname === "/healthz") {
+        sendJson(response, 200, { ok: true });
+        return;
+      }
+      if (
+        request.method === "GET" &&
+        (url.pathname === "/" || url.pathname.startsWith("/assets/"))
+      ) {
+        await sendReviewAsset(response, url.pathname);
+        return;
+      }
+      if (!authorized(request, config.apiToken)) {
+        sendJson(response, 401, { error: "unauthorized" });
+        return;
+      }
+      if (request.method === "POST" && url.pathname === "/v1/provenance") {
+        const runId = z
+          .string()
+          .regex(/^[a-z0-9][a-z0-9-]{2,62}$/)
+          .parse(url.searchParams.get("runId"));
+        const body = await readBody(request, 100 * 1024 * 1024);
+        const reference = await artifacts.put(
+          `runs/${runId}/input/provenance.jsonl`,
+          body,
+          "application/x-ndjson",
+        );
+        sendJson(response, 201, reference);
+        return;
+      }
+      if (request.method === "POST" && url.pathname === "/v1/runs") {
+        const submission = submissionSchema.parse(
+          JSON.parse((await readBody(request)).toString("utf8")),
+        );
+        const workflowInput = runRequestSchema.parse({
+          runId: submission.runId,
+          repository: submission.repository,
+          provenance: submission.provenance,
+          candidateCounts: submission.candidateCounts,
+          authoring: {
+            provider: "openai-codex",
+            model: submission.authoringModel,
+            reasoningEffort: "high",
+          },
+          version: {
+            selfbenchCommit: config.buildCommit ?? submission.selfbenchCommit,
+            executionBackend: config.execution.kind,
+            sandboxImage: config.execution.image,
+            schema: 1,
+          },
+        });
+        await client.workflow.start(selfBenchRunWorkflow, {
+          workflowId: workflowInput.runId,
+          taskQueue: config.temporal.taskQueue,
+          args: [workflowInput],
+          workflowExecutionTimeout: "14 days",
+        });
+        sendJson(response, 202, { runId: workflowInput.runId });
+        return;
+      }
+      const runMatch = /^\/v1\/runs\/([a-z0-9][a-z0-9-]{2,62})(?:\/(cancel|export))?$/.exec(
+        url.pathname,
+      );
+      if (runMatch?.[1] && request.method === "GET" && runMatch[2] === "export") {
+        const status = await queryStatus(client.workflow.getHandle(runMatch[1]));
+        if (!("export" in status) || !status.export) {
+          sendJson(response, 409, { error: "run export is not ready" });
+          return;
+        }
+        const body = await artifacts.openRead(status.export);
+        response.writeHead(200, {
+          "content-type": status.export.contentType,
+          "content-length": status.export.sizeBytes,
+          "content-disposition": `attachment; filename="selfbench-${runMatch[1]}.tar.gz"`,
+          "x-content-sha256": status.export.sha256,
+        });
+        await pipeline(body, response);
+        return;
+      }
+      if (runMatch?.[1] && request.method === "GET" && !runMatch[2]) {
+        const handle = client.workflow.getHandle(runMatch[1]);
+        const status = await queryStatus(handle);
+        sendJson(response, 200, status);
+        return;
+      }
+      if (runMatch?.[1] && request.method === "POST" && runMatch[2] === "cancel") {
+        await client.workflow.getHandle(runMatch[1]).cancel();
+        sendJson(response, 202, { runId: runMatch[1], cancellationRequested: true });
+        return;
+      }
+      if (request.method === "GET" && url.pathname === "/v1/runs") {
+        const runs: unknown[] = [];
+        for await (const execution of client.workflow.list({
+          query: "WorkflowType = 'selfBenchRunWorkflow'",
+        })) {
+          runs.push({
+            runId: execution.workflowId,
+            status: execution.status.name,
+            startedAt: execution.startTime.toISOString(),
+            closedAt: execution.closeTime?.toISOString(),
+          });
+        }
+        sendJson(response, 200, runs);
+        return;
+      }
+      sendJson(response, 404, { error: "not found" });
+    } catch (error) {
+      if (response.headersSent) {
+        response.destroy(error instanceof Error ? error : new Error(String(error)));
+        return;
+      }
+      const message = error instanceof Error ? error.message : String(error);
+      sendJson(response, error instanceof z.ZodError || error instanceof SyntaxError ? 400 : 500, {
+        error: message,
+      });
+    }
+  });
+
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(config.apiPort, config.apiHost, resolve);
+  });
+  return async () => {
+    await new Promise<void>((resolve, reject) =>
+      server.close((error) => (error ? reject(error) : resolve())),
+    );
+    await connection.close();
+  };
+}
+
+async function queryStatus(
+  handle: ReturnType<Client["workflow"]["getHandle"]>,
+): Promise<RunStatus | object> {
+  try {
+    const [status, description] = await Promise.all([handle.query(statusQuery), handle.describe()]);
+    if (description.status.name === "RUNNING" || terminalRunPhase(status.phase)) {
+      return status;
+    }
+    const phase = executionPhase(description.status.name);
+    return {
+      ...status,
+      phase,
+      ...(phase === "failed" && !status.error
+        ? { error: `Temporal workflow ${description.status.name.toLowerCase()}` }
+        : {}),
+    };
+  } catch {
+    const description = await handle.describe();
+    return {
+      runId: description.workflowId,
+      phase: executionPhase(description.status.name),
+    };
+  }
+}
+
+function terminalRunPhase(phase: RunPhase): boolean {
+  return ["complete", "blocked", "failed", "cancelled"].includes(phase);
+}
+
+function executionPhase(status: string): RunPhase {
+  switch (status) {
+    case "COMPLETED":
+      return "complete";
+    case "CANCELED":
+      return "cancelled";
+    case "RUNNING":
+      return "queued";
+    default:
+      return "failed";
+  }
+}
+
+async function readBody(request: IncomingMessage, limit = 10 * 1024 * 1024): Promise<Buffer> {
+  const chunks: Buffer[] = [];
+  let size = 0;
+  for await (const chunk of request) {
+    const value = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+    size += value.byteLength;
+    if (size > limit) {
+      throw new Error(`request body exceeds ${limit} bytes`);
+    }
+    chunks.push(value);
+  }
+  return Buffer.concat(chunks);
+}
+
+function authorized(request: IncomingMessage, token: string | undefined): boolean {
+  if (!token) {
+    return true;
+  }
+  const supplied = request.headers.authorization?.replace(/^Bearer\s+/i, "");
+  if (!supplied) {
+    return false;
+  }
+  const expectedBuffer = Buffer.from(token);
+  const suppliedBuffer = Buffer.from(supplied);
+  return (
+    expectedBuffer.length === suppliedBuffer.length &&
+    timingSafeEqual(expectedBuffer, suppliedBuffer)
+  );
+}
+
+async function sendReviewAsset(response: ServerResponse, pathname: string): Promise<void> {
+  const relativePath = pathname === "/" ? "index.html" : pathname.slice(1);
+  const root = join(dirname(fileURLToPath(import.meta.url)), "review");
+  const path = join(root, relativePath);
+  if (!path.startsWith(`${root}/`)) {
+    sendJson(response, 400, { error: "invalid asset path" });
+    return;
+  }
+  try {
+    const body = await readFile(path);
+    response.writeHead(200, {
+      "content-type": contentType(path),
+      "content-length": body.byteLength,
+      "cache-control": pathname === "/" ? "no-cache" : "public, max-age=31536000, immutable",
+    });
+    response.end(body);
+  } catch {
+    sendJson(response, 404, { error: "asset not found" });
+  }
+}
+
+function contentType(path: string): string {
+  if (path.endsWith(".html")) return "text/html; charset=utf-8";
+  if (path.endsWith(".css")) return "text/css; charset=utf-8";
+  if (path.endsWith(".js")) return "text/javascript; charset=utf-8";
+  return "application/octet-stream";
+}
+
+function sendJson(response: ServerResponse, status: number, value: unknown): void {
+  const body = `${JSON.stringify(value, null, 2)}\n`;
+  response.writeHead(status, {
+    "content-type": "application/json; charset=utf-8",
+    "content-length": Buffer.byteLength(body),
+  });
+  response.end(body);
+}

package/src/artifacts.ts

@@ -0,0 +1,357 @@
+import { createHash, randomUUID } from "node:crypto";
+import { createReadStream, createWriteStream } from "node:fs";
+import { link, mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve, sep } from "node:path";
+import { Readable, Transform } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { Storage } from "@google-cloud/storage";
+import type { SelfBenchConfig } from "./config.js";
+import type { ArtifactRef } from "./contracts.js";
+import { sha256 } from "./hash.js";
+
+export interface ArtifactStore {
+  put(key: string, value: Uint8Array, contentType: string): Promise<ArtifactRef>;
+  putFile(key: string, sourcePath: string, contentType: string): Promise<ArtifactRef>;
+  get(reference: ArtifactRef): Promise<Uint8Array>;
+  openRead(reference: ArtifactRef): Promise<Readable>;
+  getByKey(key: string): Promise<Uint8Array | undefined>;
+}
+
+export function createArtifactStore(config: SelfBenchConfig["artifact"]): ArtifactStore {
+  return config.kind === "gcs"
+    ? new GcsArtifactStore(config.bucket, config.prefix)
+    : new LocalArtifactStore(config.directory);
+}
+
+export class LocalArtifactStore implements ArtifactStore {
+  readonly #root: string;
+
+  constructor(root: string) {
+    this.#root = resolve(root);
+  }
+
+  async put(key: string, value: Uint8Array, contentType: string): Promise<ArtifactRef> {
+    const path = this.#pathFor(key);
+    await mkdir(dirname(path), { recursive: true });
+    const digest = sha256(value);
+    try {
+      await writeFile(path, value, { flag: "wx" });
+    } catch (error) {
+      const existing = await fileDigest(path).catch(() => undefined);
+      if (!existing || existing.sha256 !== digest || existing.sizeBytes !== value.byteLength) {
+        throw error;
+      }
+    }
+    return {
+      uri: `file://${path}`,
+      sha256: digest,
+      sizeBytes: value.byteLength,
+      contentType,
+    };
+  }
+
+  async putFile(key: string, sourcePath: string, contentType: string): Promise<ArtifactRef> {
+    const path = this.#pathFor(key);
+    await mkdir(dirname(path), { recursive: true });
+    const temporaryPath = `${path}.${randomUUID()}.tmp`;
+    try {
+      const digest = await copyWithDigest(sourcePath, temporaryPath);
+      try {
+        await link(temporaryPath, path);
+      } catch (error) {
+        if (!hasCode(error, "EEXIST")) {
+          throw error;
+        }
+        const existing = await fileDigest(path).catch(() => undefined);
+        if (
+          !existing ||
+          existing.sha256 !== digest.sha256 ||
+          existing.sizeBytes !== digest.sizeBytes
+        ) {
+          throw new Error(`artifact already exists with different contents: file://${path}`);
+        }
+      }
+      return {
+        uri: `file://${path}`,
+        ...digest,
+        contentType,
+      };
+    } finally {
+      await rm(temporaryPath, { force: true });
+    }
+  }
+
+  async get(reference: ArtifactRef): Promise<Uint8Array> {
+    const url = new URL(reference.uri);
+    if (url.protocol !== "file:") {
+      throw new Error(`local artifact store cannot read ${reference.uri}`);
+    }
+    const path = resolve(decodeURIComponent(url.pathname));
+    this.#assertInsideRoot(path);
+    const value = await readFile(path);
+    verifyArtifact(reference, value);
+    return value;
+  }
+
+  async openRead(reference: ArtifactRef): Promise<Readable> {
+    const path = this.#pathForReference(reference);
+    if ((await stat(path)).size !== reference.sizeBytes) {
+      throw new Error(`artifact integrity check failed: ${reference.uri}`);
+    }
+    return verifiedArtifactReadStream(reference, createReadStream(path));
+  }
+
+  async getByKey(key: string): Promise<Uint8Array | undefined> {
+    try {
+      return await readFile(this.#pathFor(key));
+    } catch (error) {
+      if (isNotFound(error)) {
+        return undefined;
+      }
+      throw error;
+    }
+  }
+
+  #pathForReference(reference: ArtifactRef): string {
+    const url = new URL(reference.uri);
+    if (url.protocol !== "file:") {
+      throw new Error(`local artifact store cannot read ${reference.uri}`);
+    }
+    const path = resolve(decodeURIComponent(url.pathname));
+    this.#assertInsideRoot(path);
+    return path;
+  }
+
+  #pathFor(key: string): string {
+    if (!key || key.startsWith("/") || key.split("/").some((part) => part === "..")) {
+      throw new Error(`unsafe artifact key: ${key}`);
+    }
+    const path = resolve(this.#root, key);
+    this.#assertInsideRoot(path);
+    return path;
+  }
+
+  #assertInsideRoot(path: string): void {
+    if (path !== this.#root && !path.startsWith(`${this.#root}${sep}`)) {
+      throw new Error(`artifact path escapes root: ${path}`);
+    }
+  }
+}
+
+export class GcsArtifactStore implements ArtifactStore {
+  readonly #storage = new Storage();
+  readonly #bucket: string;
+  readonly #prefix: string;
+
+  constructor(bucket: string, prefix: string) {
+    this.#bucket = bucket;
+    this.#prefix = prefix.replace(/^\/+|\/+$/g, "");
+  }
+
+  async put(key: string, value: Uint8Array, contentType: string): Promise<ArtifactRef> {
+    const object = this.#objectFor(key);
+    const digest = sha256(value);
+    const file = this.#storage.bucket(this.#bucket).file(object);
+    try {
+      await file.save(value, {
+        resumable: false,
+        preconditionOpts: { ifGenerationMatch: 0 },
+        metadata: {
+          contentType,
+          metadata: { sha256: digest },
+        },
+      });
+    } catch (error) {
+      const [exists] = await file.exists();
+      if (!exists) {
+        throw error;
+      }
+      const [metadata] = await file.getMetadata();
+      if (metadata.metadata?.sha256 !== digest) {
+        throw new Error(
+          `artifact already exists with different contents: gs://${this.#bucket}/${object}`,
+        );
+      }
+    }
+    return {
+      uri: `gs://${this.#bucket}/${object}`,
+      sha256: digest,
+      sizeBytes: value.byteLength,
+      contentType,
+    };
+  }
+
+  async putFile(key: string, sourcePath: string, contentType: string): Promise<ArtifactRef> {
+    const object = this.#objectFor(key);
+    const bucket = this.#storage.bucket(this.#bucket);
+    const file = bucket.file(object);
+    const snapshotDirectory = await mkdtemp(join(tmpdir(), "selfbench-artifact-"));
+    const snapshotPath = join(snapshotDirectory, "upload");
+    try {
+      const digest = await copyWithDigest(sourcePath, snapshotPath);
+      try {
+        await bucket.upload(snapshotPath, {
+          destination: object,
+          resumable: true,
+          preconditionOpts: { ifGenerationMatch: 0 },
+          metadata: {
+            contentType,
+            metadata: { sha256: digest.sha256 },
+          },
+        });
+      } catch (error) {
+        const [exists] = await file.exists();
+        if (!exists) {
+          throw error;
+        }
+        const [metadata] = await file.getMetadata();
+        if (
+          metadata.metadata?.sha256 !== digest.sha256 ||
+          Number(metadata.size) !== digest.sizeBytes
+        ) {
+          throw new Error(
+            `artifact already exists with different contents: gs://${this.#bucket}/${object}`,
+          );
+        }
+      }
+      return {
+        uri: `gs://${this.#bucket}/${object}`,
+        ...digest,
+        contentType,
+      };
+    } finally {
+      await rm(snapshotDirectory, { recursive: true, force: true });
+    }
+  }
+
+  async get(reference: ArtifactRef): Promise<Uint8Array> {
+    const match = /^gs:\/\/([^/]+)\/(.+)$/.exec(reference.uri);
+    if (!match) {
+      throw new Error(`GCS artifact store cannot read ${reference.uri}`);
+    }
+    const [, bucket, object] = match;
+    if (bucket !== this.#bucket || !object) {
+      throw new Error(`artifact is outside configured bucket: ${reference.uri}`);
+    }
+    if (this.#prefix && object !== this.#prefix && !object.startsWith(`${this.#prefix}/`)) {
+      throw new Error(`artifact is outside configured bucket: ${reference.uri}`);
+    }
+    const [value] = await this.#storage.bucket(bucket).file(object).download();
+    verifyArtifact(reference, value);
+    return value;
+  }
+
+  async openRead(reference: ArtifactRef): Promise<Readable> {
+    const file = this.#fileForReference(reference);
+    const [metadata] = await file.getMetadata();
+    if (
+      Number(metadata.size) !== reference.sizeBytes ||
+      metadata.metadata?.sha256 !== reference.sha256
+    ) {
+      throw new Error(`artifact integrity check failed: ${reference.uri}`);
+    }
+    return verifiedArtifactReadStream(reference, file.createReadStream());
+  }
+
+  async getByKey(key: string): Promise<Uint8Array | undefined> {
+    const file = this.#storage.bucket(this.#bucket).file(this.#objectFor(key));
+    const [exists] = await file.exists();
+    if (!exists) {
+      return undefined;
+    }
+    const [value] = await file.download();
+    return value;
+  }
+
+  #fileForReference(reference: ArtifactRef) {
+    const match = /^gs:\/\/([^/]+)\/(.+)$/.exec(reference.uri);
+    if (!match) {
+      throw new Error(`GCS artifact store cannot read ${reference.uri}`);
+    }
+    const [, bucket, object] = match;
+    if (bucket !== this.#bucket || !object) {
+      throw new Error(`artifact is outside configured bucket: ${reference.uri}`);
+    }
+    if (this.#prefix && object !== this.#prefix && !object.startsWith(`${this.#prefix}/`)) {
+      throw new Error(`artifact is outside configured bucket: ${reference.uri}`);
+    }
+    return this.#storage.bucket(bucket).file(object);
+  }
+
+  #objectFor(key: string): string {
+    if (!key || key.startsWith("/") || key.split("/").some((part) => part === "..")) {
+      throw new Error(`unsafe artifact key: ${key}`);
+    }
+    return this.#prefix ? `${this.#prefix}/${key}` : key;
+  }
+}
+
+export function verifiedArtifactReadStream(reference: ArtifactRef, input: Readable): Readable {
+  return Readable.from(
+    (async function* () {
+      const hash = createHash("sha256");
+      let sizeBytes = 0;
+      for await (const chunk of input) {
+        const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+        hash.update(bytes);
+        sizeBytes += bytes.byteLength;
+        yield bytes;
+      }
+      if (sizeBytes !== reference.sizeBytes || hash.digest("hex") !== reference.sha256) {
+        throw new Error(`artifact integrity check failed: ${reference.uri}`);
+      }
+    })(),
+  );
+}
+
+async function copyWithDigest(
+  sourcePath: string,
+  destinationPath: string,
+): Promise<{ sha256: string; sizeBytes: number }> {
+  const hash = createHash("sha256");
+  let sizeBytes = 0;
+  const hasher = new Transform({
+    transform(chunk: Buffer, _encoding, callback) {
+      hash.update(chunk);
+      sizeBytes += chunk.byteLength;
+      callback(undefined, chunk);
+    },
+  });
+  await pipeline(
+    createReadStream(sourcePath),
+    hasher,
+    createWriteStream(destinationPath, { flags: "wx" }),
+  );
+  return { sha256: hash.digest("hex"), sizeBytes };
+}
+
+async function fileDigest(path: string): Promise<{ sha256: string; sizeBytes: number }> {
+  const hash = createHash("sha256");
+  let sizeBytes = 0;
+  for await (const chunk of createReadStream(path)) {
+    const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+    hash.update(bytes);
+    sizeBytes += bytes.byteLength;
+  }
+  return { sha256: hash.digest("hex"), sizeBytes };
+}
+
+function hasCode(error: unknown, code: string): boolean {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    (error as { code?: unknown }).code === code
+  );
+}
+
+function isNotFound(error: unknown): boolean {
+  return hasCode(error, "ENOENT");
+}
+
+function verifyArtifact(reference: ArtifactRef, value: Uint8Array): void {
+  if (value.byteLength !== reference.sizeBytes || sha256(value) !== reference.sha256) {
+    throw new Error(`artifact integrity check failed: ${reference.uri}`);
+  }
+}