self-bench 0.3.4 → 0.3.6

package/review/src/App.tsx

@@ -0,0 +1,338 @@
+import { parsePatchFiles } from "@pierre/diffs";
+import { FileDiff } from "@pierre/diffs/react";
+import React from "react";
+import { buildCleanExport, loadExport } from "./export-loader";
+import type { ExportTask, LoadedExport, RunStatus, RunSummary } from "./types";
+
+export function App() {
+  const [runs, setRuns] = React.useState<RunSummary[]>([]);
+  const [runId, setRunId] = React.useState<string | null>(null);
+  const [status, setStatus] = React.useState<RunStatus | null>(null);
+  const [loaded, setLoaded] = React.useState<LoadedExport | null>(null);
+  const [removed, setRemoved] = React.useState<Set<string>>(new Set());
+  const [selectedTaskId, setSelectedTaskId] = React.useState<string | null>(null);
+  const [error, setError] = React.useState<string | null>(null);
+  const [busy, setBusy] = React.useState(false);
+  const [token, setToken] = React.useState("");
+  const fileInput = React.useRef<HTMLInputElement>(null);
+  const apiFetch = React.useCallback(
+    (path: string) =>
+      fetch(path, token ? { headers: { Authorization: `Bearer ${token}` } } : undefined),
+    [token],
+  );
+
+  const refreshRuns = React.useCallback(async () => {
+    try {
+      setError(null);
+      const response = await apiFetch("/v1/runs");
+      if (!response.ok) throw new Error(`Unable to list runs (${response.status})`);
+      setRuns((await response.json()) as RunSummary[]);
+    } catch (cause) {
+      setError(cause instanceof Error ? cause.message : String(cause));
+    }
+  }, [apiFetch]);
+
+  React.useEffect(() => {
+    void refreshRuns();
+  }, [refreshRuns]);
+
+  const openRun = async (nextRunId: string) => {
+    try {
+      setBusy(true);
+      setError(null);
+      setRunId(nextRunId);
+      setLoaded(null);
+      setRemoved(new Set());
+      const [statusResponse, exportResponse] = await Promise.all([
+        apiFetch(`/v1/runs/${encodeURIComponent(nextRunId)}`),
+        apiFetch(`/v1/runs/${encodeURIComponent(nextRunId)}/export`),
+      ]);
+      if (!statusResponse.ok) throw new Error(`Unable to load run (${statusResponse.status})`);
+      if (!exportResponse.ok) throw new Error(`Export is not ready (${exportResponse.status})`);
+      setStatus((await statusResponse.json()) as RunStatus);
+      setLoaded(await loadExport(await exportResponse.blob()));
+    } catch (cause) {
+      setError(cause instanceof Error ? cause.message : String(cause));
+    } finally {
+      setBusy(false);
+    }
+  };
+
+  const openFile = async (file: File) => {
+    try {
+      setBusy(true);
+      setError(null);
+      const next = await loadExport(file);
+      setLoaded(next);
+      setRunId(next.manifest.runId);
+      setStatus({
+        runId: next.manifest.runId,
+        phase: "local export",
+        requested: next.manifest.acceptedCount,
+        accepted: next.manifest.acceptedCount,
+        rejected: 0,
+        tasks: [],
+      });
+      setRemoved(new Set());
+      setSelectedTaskId(null);
+    } catch (cause) {
+      setError(cause instanceof Error ? cause.message : String(cause));
+    } finally {
+      setBusy(false);
+    }
+  };
+
+  const selectedTask =
+    loaded?.tasks.find((task) => task.taskId === selectedTaskId) ?? loaded?.tasks[0];
+  const toggleRemoved = (taskId: string) =>
+    setRemoved((current) => {
+      const next = new Set(current);
+      if (next.has(taskId)) next.delete(taskId);
+      else next.add(taskId);
+      return next;
+    });
+
+  const downloadClean = async () => {
+    if (!loaded || !runId) return;
+    const blob = await buildCleanExport(loaded, removed);
+    const url = URL.createObjectURL(blob);
+    const link = document.createElement("a");
+    link.href = url;
+    link.download = `selfbench-${runId}-cleaned.tar.gz`;
+    link.click();
+    URL.revokeObjectURL(url);
+  };
+
+  return (
+    <main className="app-shell">
+      <header className="topbar">
+        <div>
+          <p className="eyebrow">SELFBENCH / EXPORT REVIEW</p>
+          <h1>Review accepted tasks before publishing.</h1>
+        </div>
+        <div className="topbar-actions">
+          <input
+            className="token-input"
+            type="password"
+            value={token}
+            onChange={(event) => setToken(event.target.value)}
+            placeholder="API token (session only)"
+            aria-label="API token"
+          />
+          <input
+            ref={fileInput}
+            hidden
+            type="file"
+            accept=".gz,.tgz,application/gzip"
+            onChange={(event) => {
+              const file = event.target.files?.[0];
+              if (file) void openFile(file);
+            }}
+          />
+          <button type="button" className="button" onClick={() => fileInput.current?.click()}>
+            Open gzip
+          </button>
+          <button type="button" className="button" onClick={() => void refreshRuns()}>
+            Refresh runs
+          </button>
+        </div>
+      </header>
+      {error && <div className="error-banner">{error}</div>}
+      <section className="workspace">
+        <aside className="run-list surface">
+          <div className="section-heading">
+            <h2>Runs</h2>
+            <span>{runs.length}</span>
+          </div>
+          {runs.length === 0 && <p className="muted">No runs found.</p>}
+          {runs.map((run) => (
+            <button
+              type="button"
+              className={`run-row ${run.runId === runId ? "selected" : ""}`}
+              key={run.runId}
+              onClick={() => void openRun(run.runId)}
+            >
+              <strong>{run.runId}</strong>
+              <span>{run.status}</span>
+              <small>{new Date(run.startedAt).toLocaleString()}</small>
+            </button>
+          ))}
+        </aside>
+        <section className="content">
+          {!loaded && (
+            <div className="empty surface">
+              <h2>{busy ? "Loading export…" : "Select a completed run"}</h2>
+              <p>Choose a run or open a local export gzip to inspect its accepted tasks.</p>
+            </div>
+          )}
+          {loaded && status && (
+            <>
+              <div className="summary surface">
+                <div>
+                  <p className="eyebrow">{loaded.manifest.repository.url}</p>
+                  <h2>{runId}</h2>
+                  <p className="muted">
+                    {status.phase} · {loaded.manifest.acceptedCount} accepted tasks · commit{" "}
+                    {loaded.manifest.repository.commit.slice(0, 12)}
+                  </p>
+                </div>
+                <button
+                  type="button"
+                  className="button primary"
+                  onClick={() => void downloadClean()}
+                >
+                  {removed.size
+                    ? `Download cleaned export (${loaded.tasks.length - removed.size})`
+                    : "Download export copy"}
+                </button>
+              </div>
+              <div className="review-grid">
+                <div className="task-list surface">
+                  <div className="section-heading">
+                    <h2>Tasks</h2>
+                    <span>{removed.size} marked for removal</span>
+                  </div>
+                  {loaded.tasks.map((task) => (
+                    <button
+                      type="button"
+                      className={`task-row ${task.taskId === selectedTask?.taskId ? "selected" : ""} ${removed.has(task.taskId) ? "removed" : ""}`}
+                      key={task.taskId}
+                      onClick={() => setSelectedTaskId(task.taskId)}
+                    >
+                      <strong>{task.taskId}</strong>
+                      <span>{removed.has(task.taskId) ? "marked for removal" : "keep"}</span>
+                    </button>
+                  ))}
+                </div>
+                {selectedTask && (
+                  <TaskPanel
+                    task={selectedTask}
+                    removed={removed.has(selectedTask.taskId)}
+                    onToggle={() => toggleRemoved(selectedTask.taskId)}
+                  />
+                )}
+              </div>
+            </>
+          )}
+        </section>
+      </section>
+    </main>
+  );
+}
+
+function TaskPanel({
+  task,
+  removed,
+  onToggle,
+}: {
+  task: ExportTask;
+  removed: boolean;
+  onToggle: () => void;
+}) {
+  const definition = parseJson(task.textFiles.get("definition.json"));
+  return (
+    <article className="task-detail surface">
+      <div className="detail-header">
+        <div>
+          <p className="eyebrow">TASK</p>
+          <h2>{task.taskId}</h2>
+          <p className="muted">
+            {String(definition?.difficulty ?? "unknown")} · {String(definition?.repo ?? "")}
+          </p>
+        </div>
+        <button
+          type="button"
+          className={`button ${removed ? "undo" : "danger"}`}
+          onClick={onToggle}
+        >
+          {removed ? "Keep task" : "Mark for removal"}
+        </button>
+      </div>
+      <DetailBlock title="Prompt">
+        <pre>
+          {String(definition?.prompt ?? task.textFiles.get("instruction.md") ?? "No prompt")}
+        </pre>
+      </DetailBlock>
+      <DetailBlock title="Task definition">
+        <pre>{task.textFiles.get("definition.json") ?? "Missing definition.json"}</pre>
+      </DetailBlock>
+      <DetailBlock title="Gold patch">
+        <PatchBlock patch={task.textFiles.get("solution/gold.patch")} />
+      </DetailBlock>
+      <DetailBlock title="Held-out test patch">
+        <PatchBlock patch={task.textFiles.get("tests/test.patch")} />
+      </DetailBlock>
+    </article>
+  );
+}
+
+function PatchBlock({ patch }: { patch: string | undefined }) {
+  const files = React.useMemo(
+    () => (patch ? parsePatchFiles(patch).flatMap((parsed) => parsed.files) : []),
+    [patch],
+  );
+  if (!patch) return <pre>Missing patch</pre>;
+  if (!files.length) return <pre>{patch}</pre>;
+  const options = {
+    themeType: window.matchMedia("(prefers-color-scheme: dark)").matches
+      ? ("dark" as const)
+      : ("light" as const),
+    diffStyle: window.matchMedia("(max-width: 900px)").matches
+      ? ("unified" as const)
+      : ("split" as const),
+    diffIndicators: "bars" as const,
+    overflow: "scroll" as const,
+    stickyHeader: true,
+  };
+  return (
+    <DiffErrorBoundary fallback={patch}>
+      <div className="patch-files">
+        {files.map((file) => (
+          <FileDiff
+            key={`${file.prevName ?? ""}:${file.name}`}
+            fileDiff={file}
+            disableWorkerPool
+            options={options}
+          />
+        ))}
+      </div>
+    </DiffErrorBoundary>
+  );
+}
+
+class DiffErrorBoundary extends React.Component<
+  React.PropsWithChildren<{ fallback: string }>,
+  { failed: boolean }
+> {
+  state = { failed: false };
+
+  static getDerivedStateFromError() {
+    return { failed: true };
+  }
+
+  componentDidCatch(error: unknown) {
+    console.error("Pierre diff renderer failed", error);
+  }
+
+  render() {
+    return this.state.failed ? <pre>{this.props.fallback}</pre> : this.props.children;
+  }
+}
+
+function DetailBlock({ title, children }: { title: string; children: React.ReactNode }) {
+  return (
+    <section className="detail-block">
+      <h3>{title}</h3>
+      {children}
+    </section>
+  );
+}
+
+function parseJson(value: string | undefined): Record<string, unknown> | null {
+  if (!value) return null;
+  try {
+    return JSON.parse(value) as Record<string, unknown>;
+  } catch {
+    return null;
+  }
+}

package/review/src/export-loader.test.ts

@@ -0,0 +1,57 @@
+import { describe, expect, test } from "bun:test";
+import { buildCleanExport, loadExport } from "./export-loader";
+import type { LoadedExport } from "./types";
+
+describe("export review bundles", () => {
+  test("round-trips nested task archives and removes marked tasks", async () => {
+    const loaded: LoadedExport = {
+      manifest: {
+        schemaVersion: 1,
+        runId: "run-123",
+        candidateCounts: { easy: 2, medium: 0, hard: 0 },
+        repository: { url: "https://github.com/example/repo", commit: "a".repeat(40) },
+        version: { schema: 1 },
+        acceptedCount: 2,
+        tasks: [
+          { taskId: "task-a", sha256: "a".repeat(64) },
+          { taskId: "task-b", sha256: "b".repeat(64) },
+        ],
+      },
+      tasks: [],
+      taskArchives: new Map([
+        ["task-a", await taskArchive("task-a")],
+        ["task-b", await taskArchive("task-b")],
+      ]),
+    };
+    loaded.tasks = [
+      { taskId: "task-a", files: new Map(), textFiles: new Map() },
+      { taskId: "task-b", files: new Map(), textFiles: new Map() },
+    ];
+
+    const cleaned = await buildCleanExport(loaded, new Set(["task-b"]));
+    const result = await loadExport(cleaned);
+
+    expect(result.manifest.acceptedCount).toBe(1);
+    expect(result.manifest.tasks.map((task) => task.taskId)).toEqual(["task-a"]);
+    expect(result.tasks[0]?.textFiles.get("definition.json")).toBe('{"taskId":"task-a"}');
+  });
+});
+
+async function taskArchive(taskId: string): Promise<Uint8Array> {
+  const data = new TextEncoder().encode(JSON.stringify({ taskId }));
+  const header = new Uint8Array(512);
+  write(header, 0, "harbor-task/definition.json");
+  write(header, 124, data.length.toString(8).padStart(11, "0"));
+  header[156] = 48;
+  const tar = new Uint8Array(512 + Math.ceil(data.length / 512) * 512 + 1024);
+  tar.set(header);
+  tar.set(data, 512);
+  const compressed = await new Response(
+    new Blob([tar.buffer]).stream().pipeThrough(new CompressionStream("gzip")),
+  ).arrayBuffer();
+  return new Uint8Array(compressed);
+}
+
+function write(target: Uint8Array, offset: number, value: string): void {
+  target.set(new TextEncoder().encode(value), offset);
+}

package/review/src/export-loader.ts

@@ -0,0 +1,151 @@
+import type { ExportManifest, ExportTask, LoadedExport } from "./types";
+
+const textNames = new Set([
+  "definition.json",
+  "instruction.md",
+  "task.toml",
+  ".selfbench-manifest.json",
+  "solution/gold.patch",
+  "tests/test.patch",
+  "tests/dependency-setup.patch",
+]);
+
+interface TarEntry {
+  name: string;
+  data: Uint8Array;
+}
+
+export async function loadExport(input: Blob | Uint8Array): Promise<LoadedExport> {
+  const archive = input instanceof Blob ? new Uint8Array(await input.arrayBuffer()) : input;
+  const outerEntries = untar(await gunzip(archive));
+  const manifestBytes = outerEntries.find((entry) => entry.name === "manifest.json")?.data;
+  if (!manifestBytes) throw new Error("export is missing manifest.json");
+  const manifest = JSON.parse(new TextDecoder().decode(manifestBytes)) as ExportManifest;
+  const tasks: ExportTask[] = [];
+  const taskArchives = new Map<string, Uint8Array>();
+  for (const manifestTask of manifest.tasks) {
+    const name = `tasks/${manifestTask.taskId}.tar.gz`;
+    const archiveEntry = outerEntries.find((entry) => entry.name === name);
+    if (!archiveEntry) throw new Error(`export is missing ${name}`);
+    taskArchives.set(manifestTask.taskId, archiveEntry.data);
+    const entries = untar(await gunzip(archiveEntry.data));
+    const files = new Map<string, Uint8Array>();
+    const textFiles = new Map<string, string>();
+    for (const entry of entries) {
+      const relative = entry.name.replace(/^harbor-task\//, "");
+      if (!relative || entry.name.endsWith("/")) continue;
+      files.set(relative, entry.data);
+      if (textNames.has(relative) || relative.endsWith(".json")) {
+        textFiles.set(relative, new TextDecoder().decode(entry.data));
+      }
+    }
+    tasks.push({ taskId: manifestTask.taskId, files, textFiles });
+  }
+  return { manifest, tasks, taskArchives };
+}
+
+export async function buildCleanExport(
+  loaded: LoadedExport,
+  removedTaskIds: Set<string>,
+): Promise<Blob> {
+  const keptIds = new Set(
+    loaded.tasks.filter((task) => !removedTaskIds.has(task.taskId)).map((task) => task.taskId),
+  );
+  const manifest: ExportManifest = {
+    ...loaded.manifest,
+    acceptedCount: keptIds.size,
+    tasks: loaded.manifest.tasks.filter((task) => keptIds.has(task.taskId)),
+  };
+  const entries: TarEntry[] = [
+    {
+      name: "manifest.json",
+      data: new TextEncoder().encode(`${JSON.stringify(manifest, null, 2)}\n`),
+    },
+  ];
+  for (const task of loaded.tasks) {
+    if (keptIds.has(task.taskId)) {
+      const archive = loaded.taskArchives.get(task.taskId);
+      if (!archive) throw new Error(`loaded export is missing ${task.taskId}`);
+      entries.push({ name: `tasks/${task.taskId}.tar.gz`, data: archive });
+    }
+  }
+  return gzip(tar(entries));
+}
+
+async function gunzip(value: Uint8Array): Promise<Uint8Array> {
+  return transform(value, new DecompressionStream("gzip"));
+}
+
+async function gzip(value: Uint8Array): Promise<Blob> {
+  const compressed = await transform(value, new CompressionStream("gzip"));
+  return new Blob([compressed.buffer as ArrayBuffer], { type: "application/gzip" });
+}
+
+async function transform(value: Uint8Array, stream: TransformStream): Promise<Uint8Array> {
+  const output = await new Response(
+    new Blob([value.buffer as ArrayBuffer]).stream().pipeThrough(stream),
+  ).arrayBuffer();
+  return new Uint8Array(output);
+}
+
+function untar(buffer: Uint8Array): TarEntry[] {
+  const entries: TarEntry[] = [];
+  for (let offset = 0; offset + 512 <= buffer.length; ) {
+    const header = buffer.subarray(offset, offset + 512);
+    if (header.every((byte) => byte === 0)) break;
+    const name = readString(header, 0, 100);
+    const size = parseOctal(header, 124, 12);
+    const type = header[156];
+    offset += 512;
+    const data = buffer.slice(offset, offset + size);
+    if (type !== 5 && name) entries.push({ name, data });
+    offset += Math.ceil(size / 512) * 512;
+  }
+  return entries;
+}
+
+function tar(entries: TarEntry[]): Uint8Array {
+  const chunks: Uint8Array[] = [];
+  for (const entry of entries) {
+    const header = new Uint8Array(512);
+    writeString(header, 0, 100, entry.name);
+    writeOctal(header, 100, 8, 0o644);
+    writeOctal(header, 108, 8, 0);
+    writeOctal(header, 116, 8, 0);
+    writeOctal(header, 124, 12, entry.data.length);
+    writeOctal(header, 136, 12, Math.floor(Date.now() / 1000));
+    header.fill(32, 148, 156);
+    header[156] = 48;
+    writeString(header, 257, 6, "ustar\0");
+    writeString(header, 263, 2, "00");
+    const checksum = header.reduce((sum, byte) => sum + byte, 0);
+    writeString(header, 148, 8, `${checksum.toString(8).padStart(6, "0")}\0 `);
+    chunks.push(header, entry.data, new Uint8Array((512 - (entry.data.length % 512)) % 512));
+  }
+  chunks.push(new Uint8Array(1024));
+  const output = new Uint8Array(chunks.reduce((sum, chunk) => sum + chunk.length, 0));
+  let offset = 0;
+  for (const chunk of chunks) {
+    output.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return output;
+}
+
+function readString(bytes: Uint8Array, start: number, length: number): string {
+  const end = bytes.indexOf(0, start);
+  return new TextDecoder().decode(bytes.subarray(start, end < 0 ? start + length : end));
+}
+
+function parseOctal(bytes: Uint8Array, start: number, length: number): number {
+  const value = readString(bytes, start, length).trim();
+  return value ? Number.parseInt(value, 8) : 0;
+}
+
+function writeString(bytes: Uint8Array, start: number, length: number, value: string): void {
+  bytes.set(new TextEncoder().encode(value).subarray(0, length), start);
+}
+
+function writeOctal(bytes: Uint8Array, start: number, length: number, value: number): void {
+  writeString(bytes, start, length, value.toString(8).padStart(length - 1, "0"));
+}

package/review/src/main.tsx

@@ -0,0 +1,15 @@
+import React from "react";
+import { createRoot } from "react-dom/client";
+import { App } from "./App";
+import "./styles.css";
+
+const root = document.getElementById("root");
+if (!root) {
+  throw new Error("review root is missing");
+}
+
+createRoot(root).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>,
+);