self-bench 0.3.5 → 0.3.8

package/src/api.ts

@@ -1,290 +0,0 @@
-import { timingSafeEqual } from "node:crypto";
-import { readFile } from "node:fs/promises";
-import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
-import { dirname, join } from "node:path";
-import { pipeline } from "node:stream/promises";
-import { fileURLToPath } from "node:url";
-import { Client } from "@temporalio/client";
-import { z } from "zod";
-import { createArtifactStore } from "./artifacts.js";
-import type { SelfBenchConfig } from "./config.js";
-import {
-  artifactRefSchema,
-  type RunPhase,
-  type RunRequest,
-  type RunStatus,
-  repositoryRefSchema,
-  runRequestSchema,
-} from "./contracts.js";
-import { connectTemporalClient } from "./temporal/connection.js";
-import { selfBenchRunWorkflow, statusQuery } from "./temporal/workflow.js";
-
-const submissionSchema = z.object({
-  runId: z.string().regex(/^[a-z0-9][a-z0-9-]{2,62}$/),
-  repository: repositoryRefSchema,
-  provenance: artifactRefSchema,
-  candidateCounts: z.object({
-    easy: z.number().int().min(0).max(100),
-    medium: z.number().int().min(0).max(100),
-    hard: z.number().int().min(0).max(100),
-  }),
-  authoringModel: z.string().min(1).default("gpt-5.6-sol"),
-  selfbenchCommit: z.string().regex(/^[0-9a-f]{40}$/i),
-});
-
-export async function startApi(config: SelfBenchConfig): Promise<() => Promise<void>> {
-  const connection = await connectTemporalClient(config.temporal);
-  const client = new Client({ connection, namespace: config.temporal.namespace });
-  const artifacts = createArtifactStore(config.artifact);
-  const server = createServer(async (request, response) => {
-    try {
-      const url = new URL(request.url ?? "/", `http://${request.headers.host ?? "localhost"}`);
-      if (request.method === "GET" && url.pathname === "/healthz") {
-        sendJson(response, 200, { ok: true });
-        return;
-      }
-      if (
-        request.method === "GET" &&
-        (url.pathname === "/" || url.pathname.startsWith("/assets/"))
-      ) {
-        await sendReviewAsset(response, url.pathname);
-        return;
-      }
-      if (!authorized(request, config.apiToken)) {
-        sendJson(response, 401, { error: "unauthorized" });
-        return;
-      }
-      if (request.method === "POST" && url.pathname === "/v1/provenance") {
-        const runId = z
-          .string()
-          .regex(/^[a-z0-9][a-z0-9-]{2,62}$/)
-          .parse(url.searchParams.get("runId"));
-        const body = await readBody(request, 100 * 1024 * 1024);
-        const reference = await artifacts.put(
-          `runs/${runId}/input/provenance.jsonl`,
-          body,
-          "application/x-ndjson",
-        );
-        sendJson(response, 201, reference);
-        return;
-      }
-      if (request.method === "POST" && url.pathname === "/v1/runs") {
-        const workflowInput = buildRunRequest(
-          config,
-          JSON.parse((await readBody(request)).toString("utf8")),
-        );
-        await client.workflow.start(selfBenchRunWorkflow, {
-          workflowId: workflowInput.runId,
-          taskQueue: config.temporal.taskQueue,
-          args: [workflowInput],
-          workflowExecutionTimeout: "14 days",
-        });
-        sendJson(response, 202, { runId: workflowInput.runId });
-        return;
-      }
-      const runMatch = /^\/v1\/runs\/([a-z0-9][a-z0-9-]{2,62})(?:\/(cancel|export))?$/.exec(
-        url.pathname,
-      );
-      if (runMatch?.[1] && request.method === "GET" && runMatch[2] === "export") {
-        const status = await queryStatus(client.workflow.getHandle(runMatch[1]));
-        if (!("export" in status) || !status.export) {
-          sendJson(response, 409, { error: "run export is not ready" });
-          return;
-        }
-        const body = await artifacts.openRead(status.export);
-        response.writeHead(200, {
-          "content-type": status.export.contentType,
-          "content-length": status.export.sizeBytes,
-          "content-disposition": `attachment; filename="selfbench-${runMatch[1]}.tar.gz"`,
-          "x-content-sha256": status.export.sha256,
-        });
-        await pipeline(body, response);
-        return;
-      }
-      if (runMatch?.[1] && request.method === "GET" && !runMatch[2]) {
-        const handle = client.workflow.getHandle(runMatch[1]);
-        const status = await queryStatus(handle);
-        sendJson(response, 200, status);
-        return;
-      }
-      if (runMatch?.[1] && request.method === "POST" && runMatch[2] === "cancel") {
-        await client.workflow.getHandle(runMatch[1]).cancel();
-        sendJson(response, 202, { runId: runMatch[1], cancellationRequested: true });
-        return;
-      }
-      if (request.method === "GET" && url.pathname === "/v1/runs") {
-        const runs: unknown[] = [];
-        for await (const execution of client.workflow.list({
-          query: "WorkflowType = 'selfBenchRunWorkflow'",
-        })) {
-          runs.push({
-            runId: execution.workflowId,
-            status: execution.status.name,
-            startedAt: execution.startTime.toISOString(),
-            closedAt: execution.closeTime?.toISOString(),
-          });
-        }
-        sendJson(response, 200, runs);
-        return;
-      }
-      sendJson(response, 404, { error: "not found" });
-    } catch (error) {
-      if (response.headersSent) {
-        response.destroy(error instanceof Error ? error : new Error(String(error)));
-        return;
-      }
-      const message = error instanceof Error ? error.message : String(error);
-      sendJson(response, error instanceof z.ZodError || error instanceof SyntaxError ? 400 : 500, {
-        error: message,
-      });
-    }
-  });
-
-  await new Promise<void>((resolve, reject) => {
-    server.once("error", reject);
-    server.listen(config.apiPort, config.apiHost, resolve);
-  });
-  return async () => {
-    await new Promise<void>((resolve, reject) =>
-      server.close((error) => (error ? reject(error) : resolve())),
-    );
-    await connection.close();
-  };
-}
-
-export function buildRunRequest(
-  config: SelfBenchConfig,
-  submission: z.input<typeof submissionSchema>,
-): RunRequest {
-  const parsed = submissionSchema.parse(submission);
-  return runRequestSchema.parse({
-    runId: parsed.runId,
-    repository: parsed.repository,
-    provenance: parsed.provenance,
-    candidateCounts: parsed.candidateCounts,
-    authoring: {
-      provider: "openai-codex",
-      model: parsed.authoringModel,
-      reasoningEffort: "high",
-    },
-    version: {
-      selfbenchCommit: config.buildCommit ?? parsed.selfbenchCommit,
-      executionBackend: config.execution.kind,
-      harborEnvironment: config.harborEnvironment,
-      sandboxImage: config.execution.image,
-      ...(config.execution.kind === "vercel"
-        ? { sandboxTimeoutCapMs: config.execution.timeoutCapMs }
-        : {}),
-      schema: 1,
-    },
-  });
-}
-
-async function queryStatus(
-  handle: ReturnType<Client["workflow"]["getHandle"]>,
-): Promise<RunStatus | object> {
-  try {
-    const [status, description] = await Promise.all([handle.query(statusQuery), handle.describe()]);
-    if (description.status.name === "RUNNING" || terminalRunPhase(status.phase)) {
-      return status;
-    }
-    const phase = executionPhase(description.status.name);
-    return {
-      ...status,
-      phase,
-      ...(phase === "failed" && !status.error
-        ? { error: `Temporal workflow ${description.status.name.toLowerCase()}` }
-        : {}),
-    };
-  } catch {
-    const description = await handle.describe();
-    return {
-      runId: description.workflowId,
-      phase: executionPhase(description.status.name),
-    };
-  }
-}
-
-function terminalRunPhase(phase: RunPhase): boolean {
-  return ["complete", "blocked", "failed", "cancelled"].includes(phase);
-}
-
-function executionPhase(status: string): RunPhase {
-  switch (status) {
-    case "COMPLETED":
-      return "complete";
-    case "CANCELED":
-      return "cancelled";
-    case "RUNNING":
-      return "queued";
-    default:
-      return "failed";
-  }
-}
-
-async function readBody(request: IncomingMessage, limit = 10 * 1024 * 1024): Promise<Buffer> {
-  const chunks: Buffer[] = [];
-  let size = 0;
-  for await (const chunk of request) {
-    const value = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
-    size += value.byteLength;
-    if (size > limit) {
-      throw new Error(`request body exceeds ${limit} bytes`);
-    }
-    chunks.push(value);
-  }
-  return Buffer.concat(chunks);
-}
-
-function authorized(request: IncomingMessage, token: string | undefined): boolean {
-  if (!token) {
-    return true;
-  }
-  const supplied = request.headers.authorization?.replace(/^Bearer\s+/i, "");
-  if (!supplied) {
-    return false;
-  }
-  const expectedBuffer = Buffer.from(token);
-  const suppliedBuffer = Buffer.from(supplied);
-  return (
-    expectedBuffer.length === suppliedBuffer.length &&
-    timingSafeEqual(expectedBuffer, suppliedBuffer)
-  );
-}
-
-async function sendReviewAsset(response: ServerResponse, pathname: string): Promise<void> {
-  const relativePath = pathname === "/" ? "index.html" : pathname.slice(1);
-  const root = join(dirname(fileURLToPath(import.meta.url)), "review");
-  const path = join(root, relativePath);
-  if (!path.startsWith(`${root}/`)) {
-    sendJson(response, 400, { error: "invalid asset path" });
-    return;
-  }
-  try {
-    const body = await readFile(path);
-    response.writeHead(200, {
-      "content-type": contentType(path),
-      "content-length": body.byteLength,
-      "cache-control": pathname === "/" ? "no-cache" : "public, max-age=31536000, immutable",
-    });
-    response.end(body);
-  } catch {
-    sendJson(response, 404, { error: "asset not found" });
-  }
-}
-
-function contentType(path: string): string {
-  if (path.endsWith(".html")) return "text/html; charset=utf-8";
-  if (path.endsWith(".css")) return "text/css; charset=utf-8";
-  if (path.endsWith(".js")) return "text/javascript; charset=utf-8";
-  return "application/octet-stream";
-}
-
-function sendJson(response: ServerResponse, status: number, value: unknown): void {
-  const body = `${JSON.stringify(value, null, 2)}\n`;
-  response.writeHead(status, {
-    "content-type": "application/json; charset=utf-8",
-    "content-length": Buffer.byteLength(body),
-  });
-  response.end(body);
-}

package/src/artifacts.ts

@@ -1,357 +0,0 @@
-import { createHash, randomUUID } from "node:crypto";
-import { createReadStream, createWriteStream } from "node:fs";
-import { link, mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { dirname, join, resolve, sep } from "node:path";
-import { Readable, Transform } from "node:stream";
-import { pipeline } from "node:stream/promises";
-import { Storage } from "@google-cloud/storage";
-import type { SelfBenchConfig } from "./config.js";
-import type { ArtifactRef } from "./contracts.js";
-import { sha256 } from "./hash.js";
-
-export interface ArtifactStore {
-  put(key: string, value: Uint8Array, contentType: string): Promise<ArtifactRef>;
-  putFile(key: string, sourcePath: string, contentType: string): Promise<ArtifactRef>;
-  get(reference: ArtifactRef): Promise<Uint8Array>;
-  openRead(reference: ArtifactRef): Promise<Readable>;
-  getByKey(key: string): Promise<Uint8Array | undefined>;
-}
-
-export function createArtifactStore(config: SelfBenchConfig["artifact"]): ArtifactStore {
-  return config.kind === "gcs"
-    ? new GcsArtifactStore(config.bucket, config.prefix)
-    : new LocalArtifactStore(config.directory);
-}
-
-export class LocalArtifactStore implements ArtifactStore {
-  readonly #root: string;
-
-  constructor(root: string) {
-    this.#root = resolve(root);
-  }
-
-  async put(key: string, value: Uint8Array, contentType: string): Promise<ArtifactRef> {
-    const path = this.#pathFor(key);
-    await mkdir(dirname(path), { recursive: true });
-    const digest = sha256(value);
-    try {
-      await writeFile(path, value, { flag: "wx" });
-    } catch (error) {
-      const existing = await fileDigest(path).catch(() => undefined);
-      if (!existing || existing.sha256 !== digest || existing.sizeBytes !== value.byteLength) {
-        throw error;
-      }
-    }
-    return {
-      uri: `file://${path}`,
-      sha256: digest,
-      sizeBytes: value.byteLength,
-      contentType,
-    };
-  }
-
-  async putFile(key: string, sourcePath: string, contentType: string): Promise<ArtifactRef> {
-    const path = this.#pathFor(key);
-    await mkdir(dirname(path), { recursive: true });
-    const temporaryPath = `${path}.${randomUUID()}.tmp`;
-    try {
-      const digest = await copyWithDigest(sourcePath, temporaryPath);
-      try {
-        await link(temporaryPath, path);
-      } catch (error) {
-        if (!hasCode(error, "EEXIST")) {
-          throw error;
-        }
-        const existing = await fileDigest(path).catch(() => undefined);
-        if (
-          !existing ||
-          existing.sha256 !== digest.sha256 ||
-          existing.sizeBytes !== digest.sizeBytes
-        ) {
-          throw new Error(`artifact already exists with different contents: file://${path}`);
-        }
-      }
-      return {
-        uri: `file://${path}`,
-        ...digest,
-        contentType,
-      };
-    } finally {
-      await rm(temporaryPath, { force: true });
-    }
-  }
-
-  async get(reference: ArtifactRef): Promise<Uint8Array> {
-    const url = new URL(reference.uri);
-    if (url.protocol !== "file:") {
-      throw new Error(`local artifact store cannot read ${reference.uri}`);
-    }
-    const path = resolve(decodeURIComponent(url.pathname));
-    this.#assertInsideRoot(path);
-    const value = await readFile(path);
-    verifyArtifact(reference, value);
-    return value;
-  }
-
-  async openRead(reference: ArtifactRef): Promise<Readable> {
-    const path = this.#pathForReference(reference);
-    if ((await stat(path)).size !== reference.sizeBytes) {
-      throw new Error(`artifact integrity check failed: ${reference.uri}`);
-    }
-    return verifiedArtifactReadStream(reference, createReadStream(path));
-  }
-
-  async getByKey(key: string): Promise<Uint8Array | undefined> {
-    try {
-      return await readFile(this.#pathFor(key));
-    } catch (error) {
-      if (isNotFound(error)) {
-        return undefined;
-      }
-      throw error;
-    }
-  }
-
-  #pathForReference(reference: ArtifactRef): string {
-    const url = new URL(reference.uri);
-    if (url.protocol !== "file:") {
-      throw new Error(`local artifact store cannot read ${reference.uri}`);
-    }
-    const path = resolve(decodeURIComponent(url.pathname));
-    this.#assertInsideRoot(path);
-    return path;
-  }
-
-  #pathFor(key: string): string {
-    if (!key || key.startsWith("/") || key.split("/").some((part) => part === "..")) {
-      throw new Error(`unsafe artifact key: ${key}`);
-    }
-    const path = resolve(this.#root, key);
-    this.#assertInsideRoot(path);
-    return path;
-  }
-
-  #assertInsideRoot(path: string): void {
-    if (path !== this.#root && !path.startsWith(`${this.#root}${sep}`)) {
-      throw new Error(`artifact path escapes root: ${path}`);
-    }
-  }
-}
-
-export class GcsArtifactStore implements ArtifactStore {
-  readonly #storage = new Storage();
-  readonly #bucket: string;
-  readonly #prefix: string;
-
-  constructor(bucket: string, prefix: string) {
-    this.#bucket = bucket;
-    this.#prefix = prefix.replace(/^\/+|\/+$/g, "");
-  }
-
-  async put(key: string, value: Uint8Array, contentType: string): Promise<ArtifactRef> {
-    const object = this.#objectFor(key);
-    const digest = sha256(value);
-    const file = this.#storage.bucket(this.#bucket).file(object);
-    try {
-      await file.save(value, {
-        resumable: false,
-        preconditionOpts: { ifGenerationMatch: 0 },
-        metadata: {
-          contentType,
-          metadata: { sha256: digest },
-        },
-      });
-    } catch (error) {
-      const [exists] = await file.exists();
-      if (!exists) {
-        throw error;
-      }
-      const [metadata] = await file.getMetadata();
-      if (metadata.metadata?.sha256 !== digest) {
-        throw new Error(
-          `artifact already exists with different contents: gs://${this.#bucket}/${object}`,
-        );
-      }
-    }
-    return {
-      uri: `gs://${this.#bucket}/${object}`,
-      sha256: digest,
-      sizeBytes: value.byteLength,
-      contentType,
-    };
-  }
-
-  async putFile(key: string, sourcePath: string, contentType: string): Promise<ArtifactRef> {
-    const object = this.#objectFor(key);
-    const bucket = this.#storage.bucket(this.#bucket);
-    const file = bucket.file(object);
-    const snapshotDirectory = await mkdtemp(join(tmpdir(), "selfbench-artifact-"));
-    const snapshotPath = join(snapshotDirectory, "upload");
-    try {
-      const digest = await copyWithDigest(sourcePath, snapshotPath);
-      try {
-        await bucket.upload(snapshotPath, {
-          destination: object,
-          resumable: true,
-          preconditionOpts: { ifGenerationMatch: 0 },
-          metadata: {
-            contentType,
-            metadata: { sha256: digest.sha256 },
-          },
-        });
-      } catch (error) {
-        const [exists] = await file.exists();
-        if (!exists) {
-          throw error;
-        }
-        const [metadata] = await file.getMetadata();
-        if (
-          metadata.metadata?.sha256 !== digest.sha256 ||
-          Number(metadata.size) !== digest.sizeBytes
-        ) {
-          throw new Error(
-            `artifact already exists with different contents: gs://${this.#bucket}/${object}`,
-          );
-        }
-      }
-      return {
-        uri: `gs://${this.#bucket}/${object}`,
-        ...digest,
-        contentType,
-      };
-    } finally {
-      await rm(snapshotDirectory, { recursive: true, force: true });
-    }
-  }
-
-  async get(reference: ArtifactRef): Promise<Uint8Array> {
-    const match = /^gs:\/\/([^/]+)\/(.+)$/.exec(reference.uri);
-    if (!match) {
-      throw new Error(`GCS artifact store cannot read ${reference.uri}`);
-    }
-    const [, bucket, object] = match;
-    if (bucket !== this.#bucket || !object) {
-      throw new Error(`artifact is outside configured bucket: ${reference.uri}`);
-    }
-    if (this.#prefix && object !== this.#prefix && !object.startsWith(`${this.#prefix}/`)) {
-      throw new Error(`artifact is outside configured bucket: ${reference.uri}`);
-    }
-    const [value] = await this.#storage.bucket(bucket).file(object).download();
-    verifyArtifact(reference, value);
-    return value;
-  }
-
-  async openRead(reference: ArtifactRef): Promise<Readable> {
-    const file = this.#fileForReference(reference);
-    const [metadata] = await file.getMetadata();
-    if (
-      Number(metadata.size) !== reference.sizeBytes ||
-      metadata.metadata?.sha256 !== reference.sha256
-    ) {
-      throw new Error(`artifact integrity check failed: ${reference.uri}`);
-    }
-    return verifiedArtifactReadStream(reference, file.createReadStream());
-  }
-
-  async getByKey(key: string): Promise<Uint8Array | undefined> {
-    const file = this.#storage.bucket(this.#bucket).file(this.#objectFor(key));
-    const [exists] = await file.exists();
-    if (!exists) {
-      return undefined;
-    }
-    const [value] = await file.download();
-    return value;
-  }
-
-  #fileForReference(reference: ArtifactRef) {
-    const match = /^gs:\/\/([^/]+)\/(.+)$/.exec(reference.uri);
-    if (!match) {
-      throw new Error(`GCS artifact store cannot read ${reference.uri}`);
-    }
-    const [, bucket, object] = match;
-    if (bucket !== this.#bucket || !object) {
-      throw new Error(`artifact is outside configured bucket: ${reference.uri}`);
-    }
-    if (this.#prefix && object !== this.#prefix && !object.startsWith(`${this.#prefix}/`)) {
-      throw new Error(`artifact is outside configured bucket: ${reference.uri}`);
-    }
-    return this.#storage.bucket(bucket).file(object);
-  }
-
-  #objectFor(key: string): string {
-    if (!key || key.startsWith("/") || key.split("/").some((part) => part === "..")) {
-      throw new Error(`unsafe artifact key: ${key}`);
-    }
-    return this.#prefix ? `${this.#prefix}/${key}` : key;
-  }
-}
-
-export function verifiedArtifactReadStream(reference: ArtifactRef, input: Readable): Readable {
-  return Readable.from(
-    (async function* () {
-      const hash = createHash("sha256");
-      let sizeBytes = 0;
-      for await (const chunk of input) {
-        const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
-        hash.update(bytes);
-        sizeBytes += bytes.byteLength;
-        yield bytes;
-      }
-      if (sizeBytes !== reference.sizeBytes || hash.digest("hex") !== reference.sha256) {
-        throw new Error(`artifact integrity check failed: ${reference.uri}`);
-      }
-    })(),
-  );
-}
-
-async function copyWithDigest(
-  sourcePath: string,
-  destinationPath: string,
-): Promise<{ sha256: string; sizeBytes: number }> {
-  const hash = createHash("sha256");
-  let sizeBytes = 0;
-  const hasher = new Transform({
-    transform(chunk: Buffer, _encoding, callback) {
-      hash.update(chunk);
-      sizeBytes += chunk.byteLength;
-      callback(undefined, chunk);
-    },
-  });
-  await pipeline(
-    createReadStream(sourcePath),
-    hasher,
-    createWriteStream(destinationPath, { flags: "wx" }),
-  );
-  return { sha256: hash.digest("hex"), sizeBytes };
-}
-
-async function fileDigest(path: string): Promise<{ sha256: string; sizeBytes: number }> {
-  const hash = createHash("sha256");
-  let sizeBytes = 0;
-  for await (const chunk of createReadStream(path)) {
-    const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
-    hash.update(bytes);
-    sizeBytes += bytes.byteLength;
-  }
-  return { sha256: hash.digest("hex"), sizeBytes };
-}
-
-function hasCode(error: unknown, code: string): boolean {
-  return (
-    typeof error === "object" &&
-    error !== null &&
-    "code" in error &&
-    (error as { code?: unknown }).code === code
-  );
-}
-
-function isNotFound(error: unknown): boolean {
-  return hasCode(error, "ENOENT");
-}
-
-function verifyArtifact(reference: ArtifactRef, value: Uint8Array): void {
-  if (value.byteLength !== reference.sizeBytes || sha256(value) !== reference.sha256) {
-    throw new Error(`artifact integrity check failed: ${reference.uri}`);
-  }
-}