self-bench 0.3.5 → 0.3.8

package/src/provenance.ts

@@ -1,457 +0,0 @@
-import { readdir, readFile } from "node:fs/promises";
-import { basename, join } from "node:path";
-import { assertPullRequestBelongsToRepository, githubRepository } from "./github.js";
-import { runCommand } from "./process.js";
-
-export type SessionProvenanceFormat = "codex" | "claude-code" | "pi" | "generic";
-
-interface ProvenanceMessageBase {
-  readonly sessionId: string;
-  readonly messageIndex: number;
-  readonly content: string;
-}
-
-export type ProvenanceMessage =
-  | (ProvenanceMessageBase & { readonly sourceType: SessionProvenanceFormat })
-  | (ProvenanceMessageBase & {
-      readonly sourceType: "github-pull-request";
-      readonly sourcePr: number;
-      readonly sourceUrl: string;
-    });
-
-interface JsonRecord {
-  readonly [key: string]: unknown;
-}
-
-const SOURCE_ROOTS: readonly [SessionProvenanceFormat, string][] = [
-  ["pi", ".pi/agent/sessions"],
-  ["claude-code", ".claude/projects"],
-  ["codex", ".codex/sessions"],
-  ["codex", ".codex/archived_sessions"],
-];
-
-const GITHUB_PULL_REQUEST_LIMIT = 500;
-const MAX_GITHUB_BODY_LENGTH = 12_000;
-
-const INJECTED_PREFIXES = [
-  "# AGENTS.md instructions",
-  "# Review Guidelines",
-  "<environment_context>",
-  "<permissions instructions>",
-  "<collaboration_mode>",
-  "<skills_instructions>",
-  "<apps_instructions>",
-  "<plugins_instructions>",
-  "<skill name=",
-  "Base directory for this skill:",
-  "## Memory",
-] as const;
-
-export async function collectRepositoryProvenance(
-  repositoryPath: string,
-  homeDirectory: string,
-): Promise<ProvenanceMessage[]> {
-  const worktrees = await repositoryWorktrees(repositoryPath);
-  const encodedWorktrees = worktrees.flatMap((path) => [path, encodeSessionPath(path)]);
-  const messages: ProvenanceMessage[] = [];
-
-  for (const [format, relativeRoot] of SOURCE_ROOTS) {
-    const root = join(homeDirectory, relativeRoot);
-    for (const path of await listJsonFiles(root)) {
-      const raw = await readFile(path, "utf8").catch(() => undefined);
-      if (
-        !raw ||
-        !encodedWorktrees.some((needle) => raw.includes(needle) || path.includes(needle))
-      ) {
-        continue;
-      }
-      messages.push(...extractProvenanceMessages(raw, format, basename(path)));
-    }
-  }
-
-  return deduplicateMessages(messages);
-}
-
-export async function collectGitHubPullRequestProvenance(
-  repositoryUrl: string,
-): Promise<ProvenanceMessage[]> {
-  const repository = githubRepository(repositoryUrl);
-  const result = await runCommand("gh", [
-    "pr",
-    "list",
-    "--repo",
-    repository,
-    "--state",
-    "merged",
-    "--limit",
-    String(GITHUB_PULL_REQUEST_LIMIT),
-    "--json",
-    "number,title,body,url,author,isDraft,additions,deletions,changedFiles",
-  ]);
-  return extractGitHubPullRequestProvenance(result.stdout, repositoryUrl);
-}
-
-export function extractGitHubPullRequestProvenance(
-  raw: string,
-  repositoryUrl: string,
-): ProvenanceMessage[] {
-  const parsed: unknown = JSON.parse(raw);
-  if (!Array.isArray(parsed)) {
-    throw new Error("GitHub pull request response must be an array");
-  }
-  const repository = githubRepository(repositoryUrl);
-  const messages: ProvenanceMessage[] = [];
-  for (const value of parsed) {
-    if (!isRecord(value) || value.isDraft === true || !isHumanAuthor(value.author)) {
-      continue;
-    }
-    const sourcePr = positiveIntegerValue(value.number);
-    const sourceUrl = typeof value.url === "string" ? value.url : "";
-    const title = typeof value.title === "string" ? value.title.trim() : "";
-    const body = typeof value.body === "string" ? value.body.trim() : "";
-    const changedLines = nonnegativeNumber(value.additions) + nonnegativeNumber(value.deletions);
-    const changedFiles = nonnegativeNumber(value.changedFiles);
-    if (!sourcePr || !sourceUrl || !title || changedLines < 20 || changedFiles < 1) {
-      continue;
-    }
-    assertPullRequestBelongsToRepository(repositoryUrl, sourceUrl, sourcePr);
-    const content = redactSecrets(
-      body && body.length <= MAX_GITHUB_BODY_LENGTH ? `${title}\n\n${body}` : title,
-    );
-    messages.push({
-      sourceType: "github-pull-request",
-      sessionId: `github:${repository}#${sourcePr}`,
-      messageIndex: 0,
-      content,
-      sourcePr,
-      sourceUrl,
-    });
-  }
-  return messages;
-}
-
-export function assertProvenanceMatchesPullRequest(
-  message: ProvenanceMessage,
-  sourcePr: number,
-  sourceUrl: string,
-): void {
-  if (
-    message.sourceType === "github-pull-request" &&
-    (message.sourcePr !== sourcePr || message.sourceUrl !== sourceUrl)
-  ) {
-    throw new Error(
-      `pull request ${sourceUrl}#${sourcePr} does not match provenance ${message.sourceUrl}#${message.sourcePr}`,
-    );
-  }
-}
-
-export function extractProvenanceMessages(
-  raw: string,
-  format: SessionProvenanceFormat | "auto" = "auto",
-  fallbackSessionId = "unknown",
-): ProvenanceMessage[] {
-  const records = readRecords(raw);
-  const resolved = format === "auto" ? detectFormat(records) : format;
-  const sessionId = findSessionId(records, resolved) ?? fallbackSessionId;
-  const messages = extractTrace(records, resolved);
-  let messageIndex = 0;
-  const result: ProvenanceMessage[] = [];
-  for (const [role, rawContent] of messages) {
-    if (role !== "user") {
-      continue;
-    }
-    const content = redactSecrets(rawContent.trim());
-    if (!content || looksInjected(content)) {
-      continue;
-    }
-    result.push({ sourceType: resolved, sessionId, messageIndex, content });
-    messageIndex += 1;
-  }
-  return result;
-}
-
-function readRecords(raw: string): JsonRecord[] {
-  try {
-    const parsed: unknown = JSON.parse(raw);
-    if (Array.isArray(parsed)) {
-      return parsed.filter(isRecord);
-    }
-    if (isRecord(parsed)) {
-      const messages = parsed.messages;
-      return Array.isArray(messages) ? messages.filter(isRecord) : [parsed];
-    }
-    return [];
-  } catch {
-    return raw
-      .split("\n")
-      .filter((line) => line.trim())
-      .map((line) => {
-        try {
-          return JSON.parse(line) as unknown;
-        } catch {
-          return undefined;
-        }
-      })
-      .filter(isRecord);
-  }
-}
-
-function detectFormat(records: readonly JsonRecord[]): SessionProvenanceFormat {
-  if (
-    records.some((record) =>
-      ["event_msg", "response_item", "session_meta"].includes(String(record.type)),
-    )
-  ) {
-    return "codex";
-  }
-  if (
-    records.some(
-      (record) => "sessionId" in record && ["user", "assistant"].includes(String(record.type)),
-    )
-  ) {
-    return "claude-code";
-  }
-  if (records.some((record) => record.type === "message" && "parentId" in record)) {
-    return "pi";
-  }
-  return "generic";
-}
-
-function extractTrace(
-  records: readonly JsonRecord[],
-  format: SessionProvenanceFormat,
-): readonly ["user" | "assistant", string][] {
-  switch (format) {
-    case "codex":
-      return codexTrace(records);
-    case "claude-code":
-      return claudeTrace(records);
-    case "pi":
-      return nestedMessageTrace(records);
-    case "generic":
-      return genericTrace(records);
-  }
-}
-
-function codexTrace(records: readonly JsonRecord[]): readonly ["user" | "assistant", string][] {
-  const eventMessages: ["user" | "assistant", string][] = [];
-  for (const record of records) {
-    if (record.type !== "event_msg" || !isRecord(record.payload)) {
-      continue;
-    }
-    const role =
-      record.payload.type === "user_message"
-        ? "user"
-        : record.payload.type === "agent_message"
-          ? "assistant"
-          : undefined;
-    if (role && typeof record.payload.message === "string") {
-      eventMessages.push([role, record.payload.message]);
-    }
-  }
-  if (eventMessages.some(([role]) => role === "user")) {
-    return eventMessages;
-  }
-
-  const messages: ["user" | "assistant", string][] = [];
-  for (const record of records) {
-    if (
-      record.type !== "response_item" ||
-      !isRecord(record.payload) ||
-      record.payload.type !== "message"
-    ) {
-      continue;
-    }
-    const role = normalizeRole(record.payload.role);
-    const content = contentText(record.payload.content);
-    if (role && content) {
-      messages.push([role, content]);
-    }
-  }
-  return messages;
-}
-
-function claudeTrace(records: readonly JsonRecord[]): readonly ["user" | "assistant", string][] {
-  const messages: ["user" | "assistant", string][] = [];
-  for (const record of records) {
-    if (
-      !["user", "assistant"].includes(String(record.type)) ||
-      record.sourceToolAssistantUUID !== undefined
-    ) {
-      continue;
-    }
-    if (!isRecord(record.message)) {
-      continue;
-    }
-    const role = normalizeRole(record.message.role);
-    const content = contentText(record.message.content);
-    if (role && content) {
-      messages.push([role, content]);
-    }
-  }
-  return messages;
-}
-
-function nestedMessageTrace(
-  records: readonly JsonRecord[],
-): readonly ["user" | "assistant", string][] {
-  const messages: ["user" | "assistant", string][] = [];
-  for (const record of records) {
-    if (record.type !== "message" || !isRecord(record.message)) {
-      continue;
-    }
-    const role = normalizeRole(record.message.role);
-    const content = contentText(record.message.content);
-    if (role && content) {
-      messages.push([role, content]);
-    }
-  }
-  return messages;
-}
-
-function genericTrace(records: readonly JsonRecord[]): readonly ["user" | "assistant", string][] {
-  const messages: ["user" | "assistant", string][] = [];
-  for (const record of records) {
-    const directRole = normalizeRole(record.role);
-    const directContent = contentText(record.content);
-    if (directRole && directContent) {
-      messages.push([directRole, directContent]);
-      continue;
-    }
-    if (!isRecord(record.message)) {
-      continue;
-    }
-    const role = normalizeRole(record.message.role);
-    const content = contentText(record.message.content);
-    if (role && content) {
-      messages.push([role, content]);
-    }
-  }
-  return messages;
-}
-
-function contentText(content: unknown): string {
-  if (typeof content === "string") {
-    return content;
-  }
-  if (!Array.isArray(content)) {
-    return "";
-  }
-  return content
-    .flatMap((item) => {
-      if (typeof item === "string") {
-        return [item];
-      }
-      if (!isRecord(item) || !["text", "input_text"].includes(String(item.type))) {
-        return [];
-      }
-      return typeof item.text === "string" ? [item.text] : [];
-    })
-    .join("\n\n");
-}
-
-function findSessionId(
-  records: readonly JsonRecord[],
-  format: SessionProvenanceFormat,
-): string | undefined {
-  for (const record of records) {
-    if (format === "codex" && record.type === "session_meta" && isRecord(record.payload)) {
-      if (typeof record.payload.id === "string") {
-        return record.payload.id;
-      }
-    }
-    if (format === "claude-code" && typeof record.sessionId === "string") {
-      return record.sessionId;
-    }
-    if (format === "pi" && record.type === "session" && typeof record.id === "string") {
-      return record.id;
-    }
-  }
-  return undefined;
-}
-
-function normalizeRole(value: unknown): "user" | "assistant" | undefined {
-  return value === "user" || value === "assistant" ? value : undefined;
-}
-
-function looksInjected(content: string): boolean {
-  const trimmed = content.trimStart();
-  return INJECTED_PREFIXES.some((prefix) => trimmed.startsWith(prefix));
-}
-
-export function redactSecrets(value: string): string {
-  const replacements: readonly [RegExp, string][] = [
-    [
-      /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g,
-      "[REDACTED PRIVATE KEY]",
-    ],
-    [/Authorization\s*:\s*Bearer\s+[^\s,;]+/gi, "Authorization: Bearer [REDACTED]"],
-    [/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, "AWS_ACCESS_KEY_ID=[REDACTED]"],
-    [/AWS_SECRET_ACCESS_KEY\s*[:=]\s*[^\s,;]+/gi, "AWS_SECRET_ACCESS_KEY=[REDACTED]"],
-    [/\bnpm_[A-Za-z0-9]{16,}\b/g, "npm_[REDACTED]"],
-    [/\bglpat-[A-Za-z0-9_-]{16,}\b/g, "glpat-[REDACTED]"],
-    [/\b(?:password|passwd|pwd)\s*[:=]\s*[^\s,;]+/gi, "password=[REDACTED]"],
-    [/\b(?:database_url|db_url)\s*[:=]\s*[^\s]+/gi, "DATABASE_URL=[REDACTED]"],
-    [
-      /\b(?:postgres(?:ql)?|mysql|mariadb|mongodb(?:\+srv)?|redis):\/\/[^\s]+/gi,
-      "[REDACTED DATABASE URL]",
-    ],
-    [/\bdari_[A-Za-z0-9_-]{16,}/g, "dari_[REDACTED]"],
-    [/\bsk-[A-Za-z0-9_-]{16,}/g, "sk-[REDACTED]"],
-    [/\b(?:ghp|github_pat)_[A-Za-z0-9_-]{16,}/g, "github_[REDACTED]"],
-  ];
-  return replacements.reduce(
-    (result, [pattern, replacement]) => result.replace(pattern, replacement),
-    value,
-  );
-}
-
-async function repositoryWorktrees(repositoryPath: string): Promise<string[]> {
-  const result = await runCommand("git", ["-C", repositoryPath, "worktree", "list", "--porcelain"]);
-  return result.stdout
-    .split("\n")
-    .filter((line) => line.startsWith("worktree "))
-    .map((line) => line.slice("worktree ".length));
-}
-
-async function listJsonFiles(root: string): Promise<string[]> {
-  const entries = await readdir(root, { recursive: true, withFileTypes: true }).catch(() => []);
-  return entries
-    .filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl"))
-    .map((entry) => join(entry.parentPath, entry.name));
-}
-
-function encodeSessionPath(path: string): string {
-  return path.replaceAll("/", "-");
-}
-
-function deduplicateMessages(messages: readonly ProvenanceMessage[]): ProvenanceMessage[] {
-  const seen = new Set<string>();
-  return messages.filter((message) => {
-    const key = `${message.sourceType}\0${message.sessionId}\0${message.messageIndex}\0${message.content}`;
-    if (seen.has(key)) {
-      return false;
-    }
-    seen.add(key);
-    return true;
-  });
-}
-
-function isHumanAuthor(value: unknown): boolean {
-  if (!isRecord(value) || value.is_bot === true || typeof value.login !== "string") {
-    return false;
-  }
-  return !value.login.toLowerCase().endsWith("[bot]");
-}
-
-function positiveIntegerValue(value: unknown): number | undefined {
-  return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : undefined;
-}
-
-function nonnegativeNumber(value: unknown): number {
-  return typeof value === "number" && value >= 0 ? value : 0;
-}
-
-function isRecord(value: unknown): value is JsonRecord {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}

package/src/providers.ts

@@ -1,22 +0,0 @@
-export const EXECUTION_BACKENDS = ["docker", "modal", "vercel"] as const;
-export type ExecutionBackend = (typeof EXECUTION_BACKENDS)[number];
-
-export const HARBOR_ENVIRONMENTS = ["docker", "modal"] as const;
-export type HarborEnvironment = (typeof HARBOR_ENVIRONMENTS)[number];
-
-const executionBackends: ReadonlySet<string> = new Set(EXECUTION_BACKENDS);
-const harborEnvironments: ReadonlySet<string> = new Set(HARBOR_ENVIRONMENTS);
-
-export function isExecutionBackend(value: string): value is ExecutionBackend {
-  return executionBackends.has(value);
-}
-
-export function isHarborEnvironment(value: string): value is HarborEnvironment {
-  return harborEnvironments.has(value);
-}
-
-export function matchingHarborEnvironment(
-  backend: ExecutionBackend,
-): HarborEnvironment | undefined {
-  return isHarborEnvironment(backend) ? backend : undefined;
-}

package/src/reaudit-main.ts

@@ -1,198 +0,0 @@
-#!/usr/bin/env node
-
-import { access, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { basename, dirname, join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-import { parseArgs } from "node:util";
-import {
-  COUPLING_REVIEW_MODEL,
-  couplingReviewInput,
-  couplingReviewSchema,
-} from "./codex-review.js";
-import { defaultStandaloneConcurrency, loadWorkerConfig } from "./config.js";
-import {
-  buildCouplingEvidence,
-  discoverContractArtifacts,
-  resolveCouplingReview,
-  scanBaseContractArtifacts,
-} from "./coupling.js";
-import { parallelMap } from "./parallel.js";
-import { runCommand } from "./process.js";
-import { createSandboxExecutor, type SandboxExecutor } from "./sandbox.js";
-import { loadPiModelAuth } from "./subscription-auth.js";
-
-const parsed = parseArgs({
-  options: {
-    tasks: { type: "string" },
-    task: { type: "string" },
-    output: { type: "string" },
-    concurrency: { type: "string" },
-    help: { type: "boolean", short: "h" },
-  },
-  strict: true,
-});
-if (parsed.values.help) {
-  console.log(`Re-audit expanded Harbor tasks for test-to-gold coupling.
-
-Usage:
-  self-bench-reaudit --tasks DIRECTORY --output REPORT.json [options]
-
-Options:
-  --concurrency N  Concurrent Sol reviews (default: 10; Vercel: worker setting, default 4)
-  --task ID        Review only one task ID
-  -h, --help       Show this help`);
-  process.exit(0);
-}
-
-const tasksDirectory = resolve(parsed.values.tasks ?? fail("--tasks is required"));
-const outputPath = resolve(parsed.values.output ?? fail("--output is required"));
-const config = loadWorkerConfig();
-const concurrency = positiveInteger(
-  parsed.values.concurrency ?? String(defaultStandaloneConcurrency(config, 10)),
-  "--concurrency",
-);
-const sandbox = createSandboxExecutor(config.execution);
-const [taskIds, modelAuth, reviewer] = await Promise.all([
-  readdir(tasksDirectory),
-  loadPiModelAuth(),
-  readFile(join(assetRoot(), "dist/sandbox-review.bundle.js")),
-]);
-const directories = (
-  await Promise.all(
-    taskIds.map(async (taskId) => {
-      const taskDirectory = join(tasksDirectory, taskId);
-      return await access(join(taskDirectory, "instruction.md")).then(
-        () => taskDirectory,
-        () => undefined,
-      );
-    }),
-  )
-)
-  .filter((directory): directory is string => directory !== undefined)
-  .filter(
-    (directory) => parsed.values.task === undefined || basename(directory) === parsed.values.task,
-  )
-  .sort();
-if (directories.length === 0) {
-  throw new Error(parsed.values.task ? `task not found: ${parsed.values.task}` : "no tasks found");
-}
-
-const tasks = await parallelMap(directories, concurrency, async (taskDirectory) => {
-  const taskId = basename(taskDirectory);
-  console.error(`reviewing ${taskId}`);
-  try {
-    const report = await auditTask(taskDirectory, taskId, modelAuth, reviewer, sandbox);
-    console.error(`${taskId}: ${report.resolution.verdict}`);
-    return report;
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    console.error(`${taskId}: error: ${message}`);
-    return { taskId, status: "error" as const, error: message };
-  }
-});
-sandbox.close();
-const report = {
-  schemaVersion: 1,
-  reviewer: COUPLING_REVIEW_MODEL,
-  generatedAt: new Date().toISOString(),
-  sourceTasks: tasksDirectory,
-  provisional: true,
-  tasks,
-  summary: {
-    total: tasks.length,
-    clean: tasks.filter((task) => task.status === "reviewed" && task.resolution.verdict === "clean")
-      .length,
-    coupled: tasks.filter(
-      (task) => task.status === "reviewed" && task.resolution.verdict === "coupled",
-    ).length,
-    errors: tasks.filter((task) => task.status === "error").length,
-  },
-};
-await mkdir(dirname(outputPath), { recursive: true });
-await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`);
-console.log(JSON.stringify({ output: outputPath, ...report.summary }, null, 2));
-
-async function auditTask(
-  taskDirectory: string,
-  taskId: string,
-  modelAuth: { apiKey?: string; authJson?: string },
-  reviewer: Uint8Array,
-  sandbox: SandboxExecutor,
-) {
-  const scratch = await mkdtemp(join(tmpdir(), "selfbench-reaudit-"));
-  try {
-    const baseDirectory = join(scratch, "base");
-    const baseArchivePath = join(taskDirectory, "environment/repo.tar.gz");
-    await mkdir(baseDirectory);
-    await runCommand("tar", ["-xzf", baseArchivePath, "-C", baseDirectory]);
-    const [prompt, testPatch, goldPatch] = await Promise.all([
-      readFile(join(taskDirectory, "instruction.md"), "utf8"),
-      readFile(join(taskDirectory, "tests/test.patch"), "utf8"),
-      readFile(join(taskDirectory, "solution/gold.patch"), "utf8"),
-    ]);
-    const baseArtifacts = await scanBaseContractArtifacts(
-      baseDirectory,
-      scratch,
-      discoverContractArtifacts(testPatch),
-    );
-    const couplingEvidence = buildCouplingEvidence({
-      prompt,
-      testPatch,
-      goldPatch,
-      baseArtifacts,
-    });
-    const result = await sandbox.run({
-      runId: "selfbench-reaudit-v2",
-      stage: taskId,
-      timeoutMs: 15 * 60 * 1000,
-      files: [
-        { path: "/work/sandbox-review.js", contents: reviewer },
-        {
-          path: "/work/review-input.md",
-          contents: couplingReviewInput(prompt, testPatch, goldPatch, couplingEvidence),
-        },
-      ],
-      outputPaths: ["/work/review.json"],
-      secrets: {
-        ...(modelAuth.apiKey ? { OPENAI_API_KEY: modelAuth.apiKey } : {}),
-        ...(modelAuth.authJson ? { SELFBENCH_PI_AUTH_JSON: modelAuth.authJson } : {}),
-      },
-      environment: { SELFBENCH_REVIEW_OUTPUT: "/work/review.json" },
-      command: ["node", "/work/sandbox-review.js"],
-    });
-    const output = result.outputs["/work/review.json"];
-    if (result.exitCode !== 0 || !output) {
-      throw new Error(
-        `sandboxed coupling review failed in ${result.sandboxId}: ${result.stderr.trim() || result.stdout.trim()}`,
-      );
-    }
-    const review = couplingReviewSchema.parse(JSON.parse(Buffer.from(output).toString("utf8")));
-    return {
-      taskId,
-      status: "reviewed" as const,
-      sandboxId: result.sandboxId,
-      couplingEvidence,
-      review,
-      resolution: resolveCouplingReview(couplingEvidence, review),
-    };
-  } finally {
-    await rm(scratch, { recursive: true, force: true });
-  }
-}
-
-function assetRoot(): string {
-  return resolve(dirname(fileURLToPath(import.meta.url)), "..");
-}
-
-function positiveInteger(value: string | undefined, label: string): number {
-  const parsed = Number(value);
-  if (!Number.isInteger(parsed) || parsed < 1) {
-    throw new Error(`${label} must be a positive integer`);
-  }
-  return parsed;
-}
-
-function fail(message: string): never {
-  throw new Error(message);
-}