self-bench 0.3.5 → 0.3.6

package/src/provenance.ts

@@ -1,23 +1,50 @@
-import { readdir, readFile } from "node:fs/promises";
-import { basename, join } from "node:path";
+import { readdir, readFile, stat } from "node:fs/promises";
+import { basename, join, relative } from "node:path";
+import { z } from "zod";
 import { assertPullRequestBelongsToRepository, githubRepository } from "./github.js";
 import { runCommand } from "./process.js";
 
 export type SessionProvenanceFormat = "codex" | "claude-code" | "pi" | "generic";
 
-interface ProvenanceMessageBase {
+const provenanceMessageBaseSchema = z.object({
+  sessionId: z.string().min(1),
+  messageIndex: z.number().int().nonnegative(),
+  content: z.string().min(1),
+});
+
+const localProvenanceMessageSchema = provenanceMessageBaseSchema
+  .extend({
+    sourceType: z.enum(["codex", "claude-code", "pi", "generic"]),
+    sourcePr: z.number().int().positive().optional(),
+    sourceUrl: z.string().url().optional(),
+  })
+  .refine(
+    (message) => (message.sourcePr === undefined) === (message.sourceUrl === undefined),
+    "sourcePr and sourceUrl must be supplied together",
+  );
+
+export const provenanceMessageSchema = z.union([
+  localProvenanceMessageSchema,
+  provenanceMessageBaseSchema.extend({
+    sourceType: z.literal("github-pull-request"),
+    sourcePr: z.number().int().positive(),
+    sourceUrl: z.string().url(),
+  }),
+]);
+
+export type ProvenanceMessage = z.infer<typeof provenanceMessageSchema>;
+
+export interface LocalSessionMetadata {
+  readonly sourceType: SessionProvenanceFormat;
   readonly sessionId: string;
-  readonly messageIndex: number;
-  readonly content: string;
+  readonly path: string;
+  readonly modifiedAt: string;
 }
 
-export type ProvenanceMessage =
-  | (ProvenanceMessageBase & { readonly sourceType: SessionProvenanceFormat })
-  | (ProvenanceMessageBase & {
-      readonly sourceType: "github-pull-request";
-      readonly sourcePr: number;
-      readonly sourceUrl: string;
-    });
+export interface RepositoryProvenanceCollection {
+  readonly messages: readonly ProvenanceMessage[];
+  readonly sessions: readonly LocalSessionMetadata[];
+}
 
 interface JsonRecord {
   readonly [key: string]: unknown;
@@ -51,9 +78,19 @@ export async function collectRepositoryProvenance(
   repositoryPath: string,
   homeDirectory: string,
 ): Promise<ProvenanceMessage[]> {
+  return [
+    ...(await collectRepositoryProvenanceWithMetadata(repositoryPath, homeDirectory)).messages,
+  ];
+}
+
+export async function collectRepositoryProvenanceWithMetadata(
+  repositoryPath: string,
+  homeDirectory: string,
+): Promise<RepositoryProvenanceCollection> {
   const worktrees = await repositoryWorktrees(repositoryPath);
   const encodedWorktrees = worktrees.flatMap((path) => [path, encodeSessionPath(path)]);
   const messages: ProvenanceMessage[] = [];
+  const sessions: LocalSessionMetadata[] = [];
 
   for (const [format, relativeRoot] of SOURCE_ROOTS) {
     const root = join(homeDirectory, relativeRoot);
@@ -65,29 +102,54 @@ export async function collectRepositoryProvenance(
       ) {
         continue;
       }
-      messages.push(...extractProvenanceMessages(raw, format, basename(path)));
+      const extracted = extractProvenanceMessages(raw, format, basename(path));
+      if (extracted.length === 0) {
+        continue;
+      }
+      messages.push(...extracted);
+      const file = await stat(path).catch(() => undefined);
+      if (file) {
+        const sessionIds = new Set(extracted.map((message) => message.sessionId));
+        for (const sessionId of sessionIds) {
+          sessions.push({
+            sourceType: format,
+            sessionId,
+            path: displaySessionPath(path, homeDirectory),
+            modifiedAt: file.mtime.toISOString(),
+          });
+        }
+      }
     }
   }
 
-  return deduplicateMessages(messages);
+  return { messages: deduplicateMessages(messages), sessions };
 }
 
 export async function collectGitHubPullRequestProvenance(
   repositoryUrl: string,
+  token?: string,
+  signal?: AbortSignal,
 ): Promise<ProvenanceMessage[]> {
   const repository = githubRepository(repositoryUrl);
-  const result = await runCommand("gh", [
-    "pr",
-    "list",
-    "--repo",
-    repository,
-    "--state",
-    "merged",
-    "--limit",
-    String(GITHUB_PULL_REQUEST_LIMIT),
-    "--json",
-    "number,title,body,url,author,isDraft,additions,deletions,changedFiles",
-  ]);
+  const result = await runCommand(
+    "gh",
+    [
+      "pr",
+      "list",
+      "--repo",
+      repository,
+      "--state",
+      "merged",
+      "--limit",
+      String(GITHUB_PULL_REQUEST_LIMIT),
+      "--json",
+      "number,title,body,url,author,isDraft,additions,deletions,changedFiles",
+    ],
+    {
+      env: token ? { ...process.env, GH_TOKEN: token } : process.env,
+      ...(signal ? { signal } : {}),
+    },
+  );
   return extractGitHubPullRequestProvenance(result.stdout, repositoryUrl);
 }
 
@@ -130,19 +192,55 @@ export function extractGitHubPullRequestProvenance(
   return messages;
 }
 
+export function combineRunProvenance(
+  repositoryUrl: string,
+  local: readonly ProvenanceMessage[],
+  github: readonly ProvenanceMessage[],
+): ProvenanceMessage[] {
+  const explicitlyAssociatedPrs = new Set<number>();
+  for (const message of local) {
+    if (message.sourcePr === undefined || message.sourceUrl === undefined) {
+      continue;
+    }
+    assertPullRequestBelongsToRepository(repositoryUrl, message.sourceUrl, message.sourcePr);
+    explicitlyAssociatedPrs.add(message.sourcePr);
+  }
+  return [
+    ...local,
+    ...github.filter(
+      (message) =>
+        message.sourceType !== "github-pull-request" ||
+        !explicitlyAssociatedPrs.has(message.sourcePr),
+    ),
+  ];
+}
+
 export function assertProvenanceMatchesPullRequest(
   message: ProvenanceMessage,
   sourcePr: number,
   sourceUrl: string,
+  provenance: readonly ProvenanceMessage[] = [message],
 ): void {
   if (
-    message.sourceType === "github-pull-request" &&
+    (message.sourcePr !== undefined || message.sourceUrl !== undefined) &&
     (message.sourcePr !== sourcePr || message.sourceUrl !== sourceUrl)
   ) {
     throw new Error(
       `pull request ${sourceUrl}#${sourcePr} does not match provenance ${message.sourceUrl}#${message.sourcePr}`,
     );
   }
+  const hasExplicitLocalAssociation = provenance.some(
+    (item) => item.sourceType !== "github-pull-request" && item.sourcePr === sourcePr,
+  );
+  if (
+    hasExplicitLocalAssociation &&
+    message.sourceType !== "github-pull-request" &&
+    (message.sourcePr !== sourcePr || message.sourceUrl !== sourceUrl)
+  ) {
+    throw new Error(
+      `pull request ${sourceUrl}#${sourcePr} has explicit local provenance; unbound local provenance cannot be selected`,
+    );
+  }
 }
 
 export function extractProvenanceMessages(
@@ -418,13 +516,19 @@ async function listJsonFiles(root: string): Promise<string[]> {
   const entries = await readdir(root, { recursive: true, withFileTypes: true }).catch(() => []);
   return entries
     .filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl"))
-    .map((entry) => join(entry.parentPath, entry.name));
+    .map((entry) => join(entry.parentPath, entry.name))
+    .sort();
 }
 
 function encodeSessionPath(path: string): string {
   return path.replaceAll("/", "-");
 }
 
+function displaySessionPath(path: string, homeDirectory: string): string {
+  const fromHome = relative(homeDirectory, path);
+  return fromHome && !fromHome.startsWith("..") ? `~/${fromHome}` : path;
+}
+
 function deduplicateMessages(messages: readonly ProvenanceMessage[]): ProvenanceMessage[] {
   const seen = new Set<string>();
   return messages.filter((message) => {

package/src/reaudit-main.ts

@@ -3,7 +3,6 @@
 import { access, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { basename, dirname, join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
 import { parseArgs } from "node:util";
 import {
   COUPLING_REVIEW_MODEL,
@@ -19,7 +18,8 @@ import {
 } from "./coupling.js";
 import { parallelMap } from "./parallel.js";
 import { runCommand } from "./process.js";
-import { createSandboxExecutor, type SandboxExecutor } from "./sandbox.js";
+import { projectRoot } from "./project-paths.js";
+import { createSandboxExecutor, type SandboxExecutor } from "./sandbox/index.js";
 import { loadPiModelAuth } from "./subscription-auth.js";
 
 const parsed = parseArgs({
@@ -182,7 +182,7 @@ async function auditTask(
 }
 
 function assetRoot(): string {
-  return resolve(dirname(fileURLToPath(import.meta.url)), "..");
+  return projectRoot(import.meta.url);
 }
 
 function positiveInteger(value: string | undefined, label: string): number {

package/src/repair-main.ts

@@ -2,13 +2,13 @@
 
 import { access, cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { basename, dirname, join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
+import { basename, join, resolve } from "node:path";
 import { parseArgs } from "node:util";
 import { defaultStandaloneConcurrency, loadWorkerConfig } from "./config.js";
 import { parallelMap } from "./parallel.js";
 import { runCommand } from "./process.js";
-import { createSandboxExecutor } from "./sandbox.js";
+import { projectRoot } from "./project-paths.js";
+import { createSandboxExecutor } from "./sandbox/index.js";
 import { loadCodexModelAuth } from "./subscription-auth.js";
 
 const parsed = parseArgs({
@@ -189,7 +189,7 @@ function reviewReport(value: unknown): { readonly tasks: readonly ReviewTask[] }
 }
 
 function assetRoot(): string {
-  return resolve(dirname(fileURLToPath(import.meta.url)), "..");
+  return projectRoot(import.meta.url);
 }
 
 function positiveInteger(value: string | undefined, label: string): number {

package/src/{sandbox.ts → sandbox/contracts.ts}

@@ -1,9 +1,3 @@
-import type { SelfBenchWorkerConfig } from "./config.js";
-import { DockerSandboxExecutor } from "./docker-executor.js";
-import { ModalSandboxExecutor } from "./modal-executor.js";
-import { TimeoutCappedSandboxExecutor } from "./sandbox-timeout.js";
-import { VercelSandboxExecutor } from "./vercel-executor.js";
-
 export interface SandboxFile {
   readonly path: string;
   readonly contents: Uint8Array | string;
@@ -41,21 +35,18 @@ export interface SandboxResult {
   readonly outputs: Readonly<Record<string, Uint8Array>>;
 }
 
+export class SandboxExecutionError extends Error {
+  constructor(
+    message: string,
+    readonly result: SandboxResult,
+    options?: ErrorOptions,
+  ) {
+    super(message, options);
+    this.name = "SandboxExecutionError";
+  }
+}
+
 export interface SandboxExecutor {
   run(request: SandboxRequest, options?: SandboxRunOptions): Promise<SandboxResult>;
   close(): void;
 }
-
-export function createSandboxExecutor(config: SelfBenchWorkerConfig["execution"]): SandboxExecutor {
-  switch (config.kind) {
-    case "docker":
-      return new DockerSandboxExecutor(config);
-    case "modal":
-      return new ModalSandboxExecutor(config);
-    case "vercel":
-      return new TimeoutCappedSandboxExecutor(
-        new VercelSandboxExecutor(config),
-        config.timeoutCapMs,
-      );
-  }
-}

package/src/sandbox/index.ts

@@ -0,0 +1,23 @@
+import type { SelfBenchWorkerConfig } from "../config.js";
+import { DockerSandboxExecutor } from "./providers/docker/executor.js";
+import { ModalSandboxExecutor } from "./providers/modal/executor.js";
+import { VercelSandboxExecutor } from "./providers/vercel/executor.js";
+import { TimeoutCappedSandboxExecutor } from "./timeout.js";
+
+export * from "./contracts.js";
+
+import type { SandboxExecutor } from "./contracts.js";
+
+export function createSandboxExecutor(config: SelfBenchWorkerConfig["execution"]): SandboxExecutor {
+  switch (config.kind) {
+    case "docker":
+      return new DockerSandboxExecutor(config);
+    case "modal":
+      return new ModalSandboxExecutor(config);
+    case "vercel":
+      return new TimeoutCappedSandboxExecutor(
+        new VercelSandboxExecutor(config),
+        config.timeoutCapMs,
+      );
+  }
+}

package/src/{sandbox-author.ts → sandbox/programs/author.ts}

@@ -2,7 +2,7 @@
 
 import { readdir } from "node:fs/promises";
 import { join } from "node:path";
-import { compileHarborTask } from "./harbor-task.js";
+import { compileHarborTask } from "../../harbor-task.js";
 
 const [tasksRoot, repositoryDirectory, outputDirectory] = process.argv.slice(2);
 if (!tasksRoot || !repositoryDirectory || !outputDirectory) {

package/src/sandbox/programs/prepare-repair.ts

@@ -0,0 +1,44 @@
+import { access, mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { runCommand } from "../../process.js";
+
+export interface PreparedRepairTask {
+  readonly extractedDirectory: string;
+  readonly taskDirectory: string;
+  readonly repositoryDirectory: string;
+}
+
+export async function prepareRepairTask(
+  archivePath: string,
+  workDirectory = "/work",
+): Promise<PreparedRepairTask> {
+  const extractedDirectory = join(workDirectory, "task");
+  const repositoryDirectory = join(workDirectory, "repo");
+  await Promise.all([mkdir(extractedDirectory, { recursive: true }), mkdir(repositoryDirectory)]);
+  await runCommand("tar", ["-xzf", archivePath, "-C", extractedDirectory]);
+  const taskDirectory = await access(join(extractedDirectory, "instruction.md")).then(
+    () => extractedDirectory,
+    () => join(extractedDirectory, "harbor-task"),
+  );
+
+  await runCommand("tar", [
+    "-xzf",
+    join(taskDirectory, "tests/repo.tar.gz"),
+    "-C",
+    repositoryDirectory,
+  ]);
+  await runCommand("git", ["-C", repositoryDirectory, "init", "-q"]);
+  await runCommand("git", ["-C", repositoryDirectory, "config", "user.name", "SelfBench"]);
+  await runCommand("git", ["-C", repositoryDirectory, "config", "user.email", "selfbench@local"]);
+  await runCommand("git", ["-C", repositoryDirectory, "add", "-A"]);
+  await runCommand("git", ["-C", repositoryDirectory, "commit", "-qm", "base"]);
+  await runCommand("git", [
+    "-C",
+    repositoryDirectory,
+    "apply",
+    join(taskDirectory, "tests/test.patch"),
+  ]);
+  await runCommand("git", ["-C", repositoryDirectory, "add", "-N", "--all"]);
+
+  return { extractedDirectory, taskDirectory, repositoryDirectory };
+}

package/src/{sandbox-repair.ts → sandbox/programs/repair.ts}

@@ -1,25 +1,20 @@
 #!/usr/bin/env node
 
-import { access, chmod, mkdir, readFile, writeFile } from "node:fs/promises";
+import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
 import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
-import { sha256 } from "./hash.js";
-import { runCommand } from "./process.js";
-import { assertRepairPaths, patchPaths, repairPrompt } from "./repair.js";
+import { sha256 } from "../../hash.js";
+import { runCommand } from "../../process.js";
+import { assertRepairPaths, patchPaths, repairPrompt } from "../../repair.js";
+import { prepareRepairTask } from "./prepare-repair.js";
 
 const [archivePath, reviewPath, outputArchive, outputReport] = process.argv.slice(2);
 if (!archivePath || !reviewPath || !outputArchive || !outputReport) {
   throw new Error("usage: sandbox-repair TASK.tar.gz REVIEW.json OUTPUT.tar.gz OUTPUT-REPORT.json");
 }
 
-const extractedDirectory = "/work/task";
-const repositoryDirectory = "/work/repo";
-await Promise.all([mkdir(extractedDirectory, { recursive: true }), mkdir(repositoryDirectory)]);
-await runCommand("tar", ["-xzf", archivePath, "-C", extractedDirectory]);
-const taskDirectory = await access(join(extractedDirectory, "instruction.md")).then(
-  () => extractedDirectory,
-  () => join(extractedDirectory, "harbor-task"),
-);
+const { extractedDirectory, taskDirectory, repositoryDirectory } =
+  await prepareRepairTask(archivePath);
 
 const [instruction, originalPatch, review] = await Promise.all([
   readFile(join(taskDirectory, "instruction.md"), "utf8"),
@@ -31,25 +26,6 @@ const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Record<stri
 const taskId = typeof manifest.taskId === "string" ? manifest.taskId : "unknown-task";
 const allowedPaths = patchPaths(originalPatch);
 
-await runCommand("tar", [
-  "-xzf",
-  join(taskDirectory, "tests/repo.tar.gz"),
-  "-C",
-  repositoryDirectory,
-]);
-await runCommand("git", ["-C", repositoryDirectory, "init", "-q"]);
-await runCommand("git", ["-C", repositoryDirectory, "config", "user.name", "SelfBench"]);
-await runCommand("git", ["-C", repositoryDirectory, "config", "user.email", "selfbench@local"]);
-await runCommand("git", ["-C", repositoryDirectory, "add", "-A"]);
-await runCommand("git", ["-C", repositoryDirectory, "commit", "-qm", "base"]);
-await runCommand("git", [
-  "-C",
-  repositoryDirectory,
-  "apply",
-  join(taskDirectory, "tests/test.patch"),
-]);
-await runCommand("git", ["-C", repositoryDirectory, "add", "-N", "--all"]);
-
 const codexAuth = process.env.SELFBENCH_CODEX_AUTH_JSON;
 const apiKey = process.env.OPENAI_API_KEY;
 if (!apiKey && !codexAuth) {

package/src/{sandbox-review.ts → sandbox/programs/review.ts}

@@ -1,5 +1,5 @@
 import { readFile, writeFile } from "node:fs/promises";
-import { reviewCouplingWithCodex } from "./codex-review.js";
+import { reviewCouplingWithCodex } from "../../codex-review.js";
 
 const apiKey = process.env.OPENAI_API_KEY;
 const authJson = process.env.SELFBENCH_PI_AUTH_JSON;

package/src/{sandbox-validation-repair.ts → sandbox/programs/validation-repair.ts}

@@ -1,15 +1,16 @@
 #!/usr/bin/env node
 
-import { access, chmod, mkdir, readFile, writeFile } from "node:fs/promises";
+import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
 import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
-import { taskDefinitionSchema } from "./contracts.js";
-import { runCommand } from "./process.js";
+import { taskDefinitionSchema } from "../../contracts.js";
+import { runCommand } from "../../process.js";
 import {
   assertValidationRepair,
   validationRepairPaths,
   validationRepairPrompt,
-} from "./validation-repair.js";
+} from "../../validation-repair.js";
+import { prepareRepairTask } from "./prepare-repair.js";
 
 const [archivePath, definitionPath, diagnosticsPath, outputDefinition, outputPatch, outputReport] =
   process.argv.slice(2);
@@ -26,14 +27,7 @@ if (
   );
 }
 
-const extractedDirectory = "/work/task";
-const repositoryDirectory = "/work/repo";
-await Promise.all([mkdir(extractedDirectory, { recursive: true }), mkdir(repositoryDirectory)]);
-await runCommand("tar", ["-xzf", archivePath, "-C", extractedDirectory]);
-const taskDirectory = await access(join(extractedDirectory, "instruction.md")).then(
-  () => extractedDirectory,
-  () => join(extractedDirectory, "harbor-task"),
-);
+const { taskDirectory, repositoryDirectory } = await prepareRepairTask(archivePath);
 const [instruction, originalPatch, diagnostics, originalDefinitionBytes] = await Promise.all([
   readFile(join(taskDirectory, "instruction.md"), "utf8"),
   readFile(join(taskDirectory, "tests/test.patch"), "utf8"),
@@ -43,24 +37,6 @@ const [instruction, originalPatch, diagnostics, originalDefinitionBytes] = await
 const originalDefinition = taskDefinitionSchema.parse(JSON.parse(originalDefinitionBytes));
 const allowedPaths = validationRepairPaths(originalPatch);
 
-await runCommand("tar", [
-  "-xzf",
-  join(taskDirectory, "tests/repo.tar.gz"),
-  "-C",
-  repositoryDirectory,
-]);
-await runCommand("git", ["-C", repositoryDirectory, "init", "-q"]);
-await runCommand("git", ["-C", repositoryDirectory, "config", "user.name", "SelfBench"]);
-await runCommand("git", ["-C", repositoryDirectory, "config", "user.email", "selfbench@local"]);
-await runCommand("git", ["-C", repositoryDirectory, "add", "-A"]);
-await runCommand("git", ["-C", repositoryDirectory, "commit", "-qm", "base"]);
-await runCommand("git", [
-  "-C",
-  repositoryDirectory,
-  "apply",
-  join(taskDirectory, "tests/test.patch"),
-]);
-await runCommand("git", ["-C", repositoryDirectory, "add", "-N", "--all"]);
 await Promise.all([
   writeFile("/work/definition.json", `${JSON.stringify(originalDefinition, null, 2)}\n`),
   writeFile("/work/gold.patch", await readFile(join(taskDirectory, "solution/gold.patch"))),

package/src/{docker-executor.ts → sandbox/providers/docker/executor.ts}

@@ -1,14 +1,14 @@
 import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { dirname, join, relative, resolve } from "node:path";
-import type { SelfBenchConfig } from "./config.js";
-import { runCommand } from "./process.js";
+import type { SelfBenchConfig } from "../../../config.js";
+import { runCommand } from "../../../process.js";
 import type {
   SandboxExecutor,
   SandboxRequest,
   SandboxResult,
   SandboxRunOptions,
-} from "./sandbox.js";
+} from "../../contracts.js";
 
 export class DockerSandboxExecutor implements SandboxExecutor {
   readonly #config: Extract<SelfBenchConfig["execution"], { kind: "docker" }>;