self-bench 0.3.2 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/.dockerignore +0 -1
  2. package/Dockerfile +2 -1
  3. package/README.md +101 -177
  4. package/bun.lock +12 -12
  5. package/compose.yaml +4 -4
  6. package/dist/api.d.ts.map +1 -1
  7. package/dist/api.js +40 -2
  8. package/dist/api.js.map +1 -1
  9. package/dist/artifacts.d.ts +0 -1
  10. package/dist/artifacts.d.ts.map +1 -1
  11. package/dist/artifacts.js +0 -3
  12. package/dist/artifacts.js.map +1 -1
  13. package/dist/build-metadata.js +1 -1
  14. package/dist/cli.js +8 -0
  15. package/dist/cli.js.map +1 -1
  16. package/dist/codex-review.d.ts +2 -1
  17. package/dist/codex-review.d.ts.map +1 -1
  18. package/dist/codex-review.js +26 -22
  19. package/dist/codex-review.js.map +1 -1
  20. package/dist/config.d.ts +0 -1
  21. package/dist/config.d.ts.map +1 -1
  22. package/dist/config.js +0 -2
  23. package/dist/config.js.map +1 -1
  24. package/dist/contracts.d.ts +1 -1
  25. package/dist/contracts.d.ts.map +1 -1
  26. package/dist/eval-main.js +2 -2
  27. package/dist/evaluate.d.ts.map +1 -1
  28. package/dist/evaluate.js +12 -7
  29. package/dist/evaluate.js.map +1 -1
  30. package/dist/provenance.d.ts +0 -1
  31. package/dist/provenance.d.ts.map +1 -1
  32. package/dist/provenance.js.map +1 -1
  33. package/dist/reaudit-main.js +9 -6
  34. package/dist/reaudit-main.js.map +1 -1
  35. package/dist/repair-main.js +7 -4
  36. package/dist/repair-main.js.map +1 -1
  37. package/dist/review/assets/index-CZ3G759O.js +1568 -0
  38. package/dist/review/assets/index-DgSLzX8l.css +1 -0
  39. package/dist/review/assets/{mojo-Blby8dQK.js → mojo-DJz3ZmWd.js} +1 -1
  40. package/dist/review/assets/typst-BUadGCkm.js +1 -0
  41. package/dist/review/index.html +2 -2
  42. package/dist/sandbox-repair.bundle.js +8 -7
  43. package/dist/sandbox-repair.js +7 -7
  44. package/dist/sandbox-repair.js.map +1 -1
  45. package/dist/sandbox-review.bundle.js +29 -25
  46. package/dist/sandbox-review.js +5 -3
  47. package/dist/sandbox-review.js.map +1 -1
  48. package/dist/sandbox-validation-repair.bundle.js +9 -9
  49. package/dist/sandbox-validation-repair.js +11 -9
  50. package/dist/sandbox-validation-repair.js.map +1 -1
  51. package/dist/subscription-auth.d.ts +12 -1
  52. package/dist/subscription-auth.d.ts.map +1 -1
  53. package/dist/subscription-auth.js +16 -16
  54. package/dist/subscription-auth.js.map +1 -1
  55. package/dist/{activities.d.ts → temporal/activities.d.ts} +2 -3
  56. package/dist/temporal/activities.d.ts.map +1 -0
  57. package/dist/{activities.js → temporal/activities.js} +44 -33
  58. package/dist/temporal/activities.js.map +1 -0
  59. package/dist/{temporal.d.ts → temporal/connection.d.ts} +2 -2
  60. package/dist/temporal/connection.d.ts.map +1 -0
  61. package/dist/{temporal.js → temporal/connection.js} +1 -1
  62. package/dist/temporal/connection.js.map +1 -0
  63. package/dist/{worker-main.d.ts.map → temporal/worker-main.d.ts.map} +1 -1
  64. package/dist/{worker-main.js → temporal/worker-main.js} +5 -9
  65. package/dist/temporal/worker-main.js.map +1 -0
  66. package/dist/{workflow.d.ts → temporal/workflow.d.ts} +1 -1
  67. package/dist/temporal/workflow.d.ts.map +1 -0
  68. package/dist/temporal/workflow.js.map +1 -0
  69. package/docs/evaluations.md +13 -12
  70. package/docs/operations.md +10 -19
  71. package/package.json +6 -6
  72. package/src/api.ts +40 -2
  73. package/src/artifacts.ts +0 -4
  74. package/src/cli.ts +9 -0
  75. package/src/codex-review.ts +63 -54
  76. package/src/config.ts +0 -3
  77. package/src/contracts.ts +0 -5
  78. package/src/eval-main.ts +2 -2
  79. package/src/evaluate.ts +13 -8
  80. package/src/provenance.ts +0 -1
  81. package/src/reaudit-main.ts +9 -6
  82. package/src/repair-main.ts +7 -4
  83. package/src/sandbox-repair.ts +7 -11
  84. package/src/sandbox-review.ts +5 -3
  85. package/src/sandbox-validation-repair.ts +13 -10
  86. package/src/subscription-auth.ts +30 -17
  87. package/src/{activities.ts → temporal/activities.ts} +45 -40
  88. package/src/{temporal.ts → temporal/connection.ts} +1 -1
  89. package/src/{worker-main.ts → temporal/worker-main.ts} +5 -9
  90. package/src/{workflow.ts → temporal/workflow.ts} +10 -10
  91. package/dist/activities.d.ts.map +0 -1
  92. package/dist/activities.js.map +0 -1
  93. package/dist/extensions/review.d.ts +0 -3
  94. package/dist/extensions/review.d.ts.map +0 -1
  95. package/dist/extensions/review.js +0 -44
  96. package/dist/extensions/review.js.map +0 -1
  97. package/dist/review/assets/index-Dq-6DFzI.css +0 -1
  98. package/dist/review/assets/index-DurW_RD7.js +0 -1567
  99. package/dist/review/assets/typst-DHCkPAjA.js +0 -1
  100. package/dist/temporal.d.ts.map +0 -1
  101. package/dist/temporal.js.map +0 -1
  102. package/dist/worker-main.js.map +0 -1
  103. package/dist/workflow.d.ts.map +0 -1
  104. package/dist/workflow.js.map +0 -1
  105. package/src/extensions/review.ts +0 -54
  106. /package/dist/{worker-main.d.ts → temporal/worker-main.d.ts} +0 -0
  107. /package/dist/{workflow.js → temporal/workflow.js} +0 -0
package/src/artifacts.ts CHANGED
@@ -355,7 +355,3 @@ function verifyArtifact(reference: ArtifactRef, value: Uint8Array): void {
355
355
  throw new Error(`artifact integrity check failed: ${reference.uri}`);
356
356
  }
357
357
  }
358
-
359
- export async function artifactSize(path: string): Promise<number> {
360
- return (await stat(path)).size;
361
- }
package/src/cli.ts CHANGED
@@ -22,6 +22,9 @@ switch (command) {
22
22
  case "run":
23
23
  await run(rest);
24
24
  break;
25
+ case "down":
26
+ await down();
27
+ break;
25
28
  case "status":
26
29
  await passthrough("GET", `/v1/runs/${requiredArgument(rest, "run ID")}`);
27
30
  break;
@@ -97,6 +100,11 @@ async function up(args: string[]): Promise<void> {
97
100
  console.log(`SelfBench is running with the ${backend} backend at http://127.0.0.1:8080`);
98
101
  }
99
102
 
103
+ async function down(): Promise<void> {
104
+ const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
105
+ await runCommand("docker", ["compose", "--file", resolve(projectRoot, "compose.yaml"), "down"]);
106
+ }
107
+
100
108
  async function run(args: string[]): Promise<void> {
101
109
  const parsed = parseArgs({
102
110
  args,
@@ -333,6 +341,7 @@ function printHelp(): void {
333
341
 
334
342
  Usage:
335
343
  self-bench up [--backend docker|modal] [--modal-config PATH]
344
+ self-bench down
336
345
  self-bench run --repo PATH [--easy-count N] [--medium-count N] [--hard-count N]
337
346
  [--model MODEL] [--run-id ID] [--wait] [--output OUTPUT.tar.gz]
338
347
  self-bench status RUN_ID
@@ -28,45 +28,54 @@ export const couplingReviewSchema = z.object({
28
28
  export type CouplingReview = z.infer<typeof couplingReviewSchema>;
29
29
 
30
30
  export async function reviewCouplingWithCodex(input: {
31
- readonly authJson: string;
31
+ readonly apiKey?: string;
32
+ readonly authJson?: string;
32
33
  readonly prompt: string;
33
34
  readonly signal?: AbortSignal;
34
35
  readonly fetch?: typeof fetch;
35
36
  }): Promise<CouplingReview> {
36
- const credential = parseCredential(input.authJson);
37
- const accountId = accountIdFromToken(credential.access);
37
+ const subscription =
38
+ !input.apiKey && input.authJson ? parseCredential(input.authJson) : undefined;
39
+ if (!input.apiKey && !subscription) {
40
+ throw new Error("OpenAI model authentication is required");
41
+ }
38
42
  const request = input.fetch ?? fetch;
39
- const response = await request("https://chatgpt.com/backend-api/codex/responses", {
40
- method: "POST",
41
- headers: {
42
- Authorization: `Bearer ${credential.access}`,
43
- "ChatGPT-Account-Id": accountId,
44
- Originator: "selfbench",
45
- "OpenAI-Beta": "responses=experimental",
46
- Accept: "text/event-stream",
47
- "Content-Type": "application/json",
43
+ const response = await request(
44
+ subscription
45
+ ? "https://chatgpt.com/backend-api/codex/responses"
46
+ : "https://api.openai.com/v1/responses",
47
+ {
48
+ method: "POST",
49
+ headers: {
50
+ Authorization: `Bearer ${subscription?.access ?? input.apiKey}`,
51
+ ...(subscription ? { "ChatGPT-Account-Id": accountIdFromToken(subscription.access) } : {}),
52
+ Originator: "selfbench",
53
+ "OpenAI-Beta": "responses=experimental",
54
+ Accept: "text/event-stream",
55
+ "Content-Type": "application/json",
56
+ },
57
+ body: JSON.stringify({
58
+ model: COUPLING_REVIEW_MODEL,
59
+ store: false,
60
+ stream: true,
61
+ instructions:
62
+ "You are an independent benchmark-quality reviewer. Analyze the supplied evidence and call submit_review exactly once.",
63
+ input: [
64
+ {
65
+ role: "user",
66
+ content: [{ type: "input_text", text: input.prompt }],
67
+ },
68
+ ],
69
+ reasoning: { effort: "high", summary: "auto" },
70
+ text: { verbosity: "low" },
71
+ include: ["reasoning.encrypted_content"],
72
+ tools: [reviewTool()],
73
+ tool_choice: { type: "function", name: "submit_review" },
74
+ parallel_tool_calls: false,
75
+ }),
76
+ ...(input.signal ? { signal: input.signal } : {}),
48
77
  },
49
- body: JSON.stringify({
50
- model: COUPLING_REVIEW_MODEL,
51
- store: false,
52
- stream: true,
53
- instructions:
54
- "You are an independent benchmark-quality reviewer. Analyze the supplied evidence and call submit_review exactly once.",
55
- input: [
56
- {
57
- role: "user",
58
- content: [{ type: "input_text", text: input.prompt }],
59
- },
60
- ],
61
- reasoning: { effort: "high", summary: "auto" },
62
- text: { verbosity: "low" },
63
- include: ["reasoning.encrypted_content"],
64
- tools: [reviewTool()],
65
- tool_choice: { type: "function", name: "submit_review" },
66
- parallel_tool_calls: false,
67
- }),
68
- ...(input.signal ? { signal: input.signal } : {}),
69
- });
78
+ );
70
79
  const responseText = await response.text();
71
80
  if (!response.ok) {
72
81
  throw new Error(
@@ -76,6 +85,27 @@ export async function reviewCouplingWithCodex(input: {
76
85
  return parseCodexReviewEvents(responseText);
77
86
  }
78
87
 
88
+ function parseCredential(value: string): { access: string } {
89
+ const parsed = JSON.parse(value) as unknown;
90
+ const credential = isRecord(parsed) ? parsed["openai-codex"] : undefined;
91
+ if (!isRecord(credential) || typeof credential.access !== "string") {
92
+ throw new Error("Pi auth does not contain an OpenAI Codex subscription access token");
93
+ }
94
+ return { access: credential.access };
95
+ }
96
+
97
+ function accountIdFromToken(token: string): string {
98
+ const payload = JSON.parse(
99
+ Buffer.from(token.split(".")[1] ?? "", "base64url").toString("utf8"),
100
+ ) as unknown;
101
+ const auth = isRecord(payload) ? payload["https://api.openai.com/auth"] : undefined;
102
+ const accountId = isRecord(auth) ? auth.chatgpt_account_id : undefined;
103
+ if (typeof accountId !== "string" || !accountId) {
104
+ throw new Error("OpenAI Codex subscription token has no ChatGPT account ID");
105
+ }
106
+ return accountId;
107
+ }
108
+
79
109
  export function parseCodexReviewEvents(value: string): CouplingReview {
80
110
  for (const line of value.split("\n")) {
81
111
  if (!line.startsWith("data: ")) {
@@ -148,27 +178,6 @@ ${goldPatch}
148
178
  `;
149
179
  }
150
180
 
151
- function parseCredential(value: string): { access: string } {
152
- const parsed = JSON.parse(value) as unknown;
153
- const credential = isRecord(parsed) ? parsed["openai-codex"] : undefined;
154
- if (!isRecord(credential) || typeof credential.access !== "string") {
155
- throw new Error("Pi auth does not contain an OpenAI Codex subscription access token");
156
- }
157
- return { access: credential.access };
158
- }
159
-
160
- function accountIdFromToken(token: string): string {
161
- const payload = JSON.parse(
162
- Buffer.from(token.split(".")[1] ?? "", "base64url").toString("utf8"),
163
- ) as unknown;
164
- const auth = isRecord(payload) ? payload["https://api.openai.com/auth"] : undefined;
165
- const accountId = isRecord(auth) ? auth.chatgpt_account_id : undefined;
166
- if (typeof accountId !== "string" || !accountId) {
167
- throw new Error("OpenAI Codex subscription token has no ChatGPT account ID");
168
- }
169
- return accountId;
170
- }
171
-
172
181
  function reviewTool(): Record<string, unknown> {
173
182
  return {
174
183
  type: "function",
package/src/config.ts CHANGED
@@ -3,7 +3,6 @@ import { z } from "zod";
3
3
  const environmentSchema = z.object({
4
4
  SELFBENCH_API_HOST: z.string().default("127.0.0.1"),
5
5
  SELFBENCH_API_PORT: z.coerce.number().int().min(1).max(65535).default(8080),
6
- SELFBENCH_API_URL: z.string().url().default("http://127.0.0.1:8080"),
7
6
  SELFBENCH_API_TOKEN: z.string().min(1).optional(),
8
7
  SELFBENCH_ARTIFACT_BACKEND: z.enum(["local", "gcs"]).default("local"),
9
8
  SELFBENCH_ARTIFACT_DIR: z.string().default(".selfbench/artifacts"),
@@ -36,7 +35,6 @@ const environmentSchema = z.object({
36
35
  export interface SelfBenchConfig {
37
36
  readonly apiHost: string;
38
37
  readonly apiPort: number;
39
- readonly apiUrl: string;
40
38
  readonly apiToken?: string;
41
39
  readonly buildCommit?: string;
42
40
  readonly activityConcurrency: number;
@@ -81,7 +79,6 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBe
81
79
  return {
82
80
  apiHost: value.SELFBENCH_API_HOST,
83
81
  apiPort: value.SELFBENCH_API_PORT,
84
- apiUrl: value.SELFBENCH_API_URL,
85
82
  ...(value.SELFBENCH_API_TOKEN ? { apiToken: value.SELFBENCH_API_TOKEN } : {}),
86
83
  ...(value.SELFBENCH_BUILD_COMMIT
87
84
  ? { buildCommit: value.SELFBENCH_BUILD_COMMIT.toLowerCase() }
package/src/contracts.ts CHANGED
@@ -130,15 +130,10 @@ export type RunPhase =
130
130
  | "queued"
131
131
  | "discovering"
132
132
  | "authoring"
133
- | "validating"
134
- | "reviewing"
135
- | "repairing"
136
- | "auditing"
137
133
  | "exporting"
138
134
  | "complete"
139
135
  | "blocked"
140
136
  | "failed"
141
- | "cancelling"
142
137
  | "cancelled";
143
138
 
144
139
  export interface TaskProgress {
package/src/eval-main.ts CHANGED
@@ -18,7 +18,7 @@ const parsed = parseArgs({
18
18
  strict: true,
19
19
  });
20
20
  if (parsed.values.help) {
21
- console.log(`Run Harbor tasks through the fixed Codex subscription model matrix.
21
+ console.log(`Run Harbor tasks through the fixed Codex model matrix.
22
22
 
23
23
  Usage:
24
24
  self-bench-eval (--export FILE.tar.gz | --tasks DIRECTORY) --jobs DIRECTORY [options]
@@ -29,7 +29,7 @@ Options:
29
29
  --environment docker|modal Execution environment (default: modal)
30
30
  --concurrency N Concurrent trials (default: 3)
31
31
  --model MODEL Run only this model; may be repeated
32
- --auth FILE Codex ChatGPT auth.json path
32
+ --auth FILE Optional Codex ChatGPT auth.json fallback
33
33
  -h, --help Show this help`);
34
34
  process.exit(0);
35
35
  }
package/src/evaluate.ts CHANGED
@@ -4,7 +4,7 @@ import { basename, join, resolve } from "node:path";
4
4
  import { archiveIncompleteHarborJob, tryReadHarborJobResult } from "./harbor-results.js";
5
5
  import { parallelMap } from "./parallel.js";
6
6
  import { runCommand } from "./process.js";
7
- import { assertCodexSubscriptionAuth } from "./subscription-auth.js";
7
+ import { assertCodexSubscriptionAuth, openAiApiKey } from "./subscription-auth.js";
8
8
 
9
9
  export const MATRIX_MODELS = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"] as const;
10
10
  export type MatrixModel = (typeof MATRIX_MODELS)[number];
@@ -40,8 +40,11 @@ export async function runMatrix(options: MatrixOptions): Promise<readonly Matrix
40
40
  const tasksDirectory = join(root, "tasks");
41
41
  await mkdir(root, { recursive: true });
42
42
  const taskDirectories = await resolveMatrixTasks(options, tasksDirectory);
43
+ const apiKey = openAiApiKey();
43
44
  const authPath = resolve(options.authPath ?? join(homedir(), ".codex/auth.json"));
44
- await assertSubscriptionAuth(authPath);
45
+ if (!apiKey) {
46
+ await assertSubscriptionAuth(authPath);
47
+ }
45
48
  const models = options.models ?? MATRIX_MODELS;
46
49
  if (models.length < 1) {
47
50
  throw new Error("provide at least one evaluation model");
@@ -55,7 +58,7 @@ export async function runMatrix(options: MatrixOptions): Promise<readonly Matrix
55
58
  jobsDirectory: root,
56
59
  harborPath: options.harborPath ?? "harbor",
57
60
  environment: options.environment ?? "modal",
58
- authPath,
61
+ ...(!apiKey ? { authPath } : {}),
59
62
  ...item,
60
63
  });
61
64
  completed += 1;
@@ -69,7 +72,7 @@ export async function runMatrix(options: MatrixOptions): Promise<readonly Matrix
69
72
  schemaVersion: 1,
70
73
  agent: "codex",
71
74
  agentVersion: CODEX_VERSION,
72
- auth: "codex-subscription",
75
+ auth: apiKey ? "openai-api-key" : "codex-subscription",
73
76
  reasoningEffort: "high",
74
77
  models,
75
78
  taskCount: taskDirectories.length,
@@ -112,7 +115,7 @@ async function runTrial(input: {
112
115
  readonly jobsDirectory: string;
113
116
  readonly harborPath: string;
114
117
  readonly environment: "docker" | "modal";
115
- readonly authPath: string;
118
+ readonly authPath?: string;
116
119
  }): Promise<MatrixTrialSummary> {
117
120
  const taskId = basename(input.taskDirectory);
118
121
  const jobName = `${taskId}-${input.model}`.toLowerCase().replace(/[^a-z0-9_.-]/g, "-");
@@ -123,9 +126,11 @@ async function runTrial(input: {
123
126
  await archiveIncompleteHarborJob(input.jobsDirectory, jobName);
124
127
 
125
128
  const environment = { ...process.env };
126
- delete environment.OPENAI_API_KEY;
127
- environment.CODEX_FORCE_AUTH_JSON = "1";
128
- environment.CODEX_AUTH_JSON_PATH = input.authPath;
129
+ if (input.authPath) {
130
+ delete environment.OPENAI_API_KEY;
131
+ environment.CODEX_FORCE_AUTH_JSON = "1";
132
+ environment.CODEX_AUTH_JSON_PATH = input.authPath;
133
+ }
129
134
  let result: Awaited<ReturnType<typeof runCommand>>;
130
135
  try {
131
136
  result = await runCommand(
package/src/provenance.ts CHANGED
@@ -4,7 +4,6 @@ import { assertPullRequestBelongsToRepository, githubRepository } from "./github
4
4
  import { runCommand } from "./process.js";
5
5
 
6
6
  export type SessionProvenanceFormat = "codex" | "claude-code" | "pi" | "generic";
7
- export type ProvenanceFormat = SessionProvenanceFormat | "github-pull-request";
8
7
 
9
8
  interface ProvenanceMessageBase {
10
9
  readonly sessionId: string;
@@ -20,7 +20,7 @@ import {
20
20
  import { parallelMap } from "./parallel.js";
21
21
  import { runCommand } from "./process.js";
22
22
  import { createSandboxExecutor, type SandboxExecutor } from "./sandbox.js";
23
- import { loadPiSubscriptionAuth } from "./subscription-auth.js";
23
+ import { loadPiModelAuth } from "./subscription-auth.js";
24
24
 
25
25
  const parsed = parseArgs({
26
26
  options: {
@@ -50,9 +50,9 @@ const outputPath = resolve(parsed.values.output ?? fail("--output is required"))
50
50
  const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
51
51
  const config = loadConfig();
52
52
  const sandbox = createSandboxExecutor(config.execution);
53
- const [taskIds, piAuth, reviewer] = await Promise.all([
53
+ const [taskIds, modelAuth, reviewer] = await Promise.all([
54
54
  readdir(tasksDirectory),
55
- loadPiSubscriptionAuth(),
55
+ loadPiModelAuth(),
56
56
  readFile(join(assetRoot(), "dist/sandbox-review.bundle.js")),
57
57
  ]);
58
58
  const directories = (
@@ -79,7 +79,7 @@ const tasks = await parallelMap(directories, concurrency, async (taskDirectory)
79
79
  const taskId = basename(taskDirectory);
80
80
  console.error(`reviewing ${taskId}`);
81
81
  try {
82
- const report = await auditTask(taskDirectory, taskId, piAuth, reviewer, sandbox);
82
+ const report = await auditTask(taskDirectory, taskId, modelAuth, reviewer, sandbox);
83
83
  console.error(`${taskId}: ${report.resolution.verdict}`);
84
84
  return report;
85
85
  } catch (error) {
@@ -113,7 +113,7 @@ console.log(JSON.stringify({ output: outputPath, ...report.summary }, null, 2));
113
113
  async function auditTask(
114
114
  taskDirectory: string,
115
115
  taskId: string,
116
- piAuth: string,
116
+ modelAuth: { apiKey?: string; authJson?: string },
117
117
  reviewer: Uint8Array,
118
118
  sandbox: SandboxExecutor,
119
119
  ) {
@@ -151,7 +151,10 @@ async function auditTask(
151
151
  },
152
152
  ],
153
153
  outputPaths: ["/work/review.json"],
154
- secrets: { SELFBENCH_PI_AUTH_JSON: piAuth },
154
+ secrets: {
155
+ ...(modelAuth.apiKey ? { OPENAI_API_KEY: modelAuth.apiKey } : {}),
156
+ ...(modelAuth.authJson ? { SELFBENCH_PI_AUTH_JSON: modelAuth.authJson } : {}),
157
+ },
155
158
  environment: { SELFBENCH_REVIEW_OUTPUT: "/work/review.json" },
156
159
  command: ["node", "/work/sandbox-review.js"],
157
160
  });
@@ -9,7 +9,7 @@ import { loadConfig } from "./config.js";
9
9
  import { parallelMap } from "./parallel.js";
10
10
  import { runCommand } from "./process.js";
11
11
  import { createSandboxExecutor } from "./sandbox.js";
12
- import { loadCodexSubscriptionAuth } from "./subscription-auth.js";
12
+ import { loadCodexModelAuth } from "./subscription-auth.js";
13
13
 
14
14
  const parsed = parseArgs({
15
15
  options: {
@@ -79,8 +79,8 @@ for (const source of sourceTasks) {
79
79
 
80
80
  const config = loadConfig();
81
81
  const sandbox = createSandboxExecutor(config.execution);
82
- const [authJson, repairer] = await Promise.all([
83
- loadCodexSubscriptionAuth(),
82
+ const [modelAuth, repairer] = await Promise.all([
83
+ loadCodexModelAuth(),
84
84
  readFile(join(assetRoot(), "dist/sandbox-repair.bundle.js")),
85
85
  ]);
86
86
  const results = await parallelMap([...coupled.values()], concurrency, async (taskReview) => {
@@ -103,7 +103,10 @@ const results = await parallelMap([...coupled.values()], concurrency, async (tas
103
103
  { path: "/work/sandbox-repair.js", contents: repairer },
104
104
  ],
105
105
  outputPaths: ["/work/repaired-task.tar.gz", "/work/repair-report.json"],
106
- secrets: { SELFBENCH_CODEX_AUTH_JSON: authJson },
106
+ secrets: {
107
+ ...(modelAuth.apiKey ? { OPENAI_API_KEY: modelAuth.apiKey } : {}),
108
+ ...(modelAuth.authJson ? { SELFBENCH_CODEX_AUTH_JSON: modelAuth.authJson } : {}),
109
+ },
107
110
  environment: { SELFBENCH_REPAIR_MODEL: parsed.values.model ?? "gpt-5.6-sol" },
108
111
  command: [
109
112
  "node",
@@ -50,13 +50,15 @@ await runCommand("git", [
50
50
  ]);
51
51
  await runCommand("git", ["-C", repositoryDirectory, "add", "-N", "--all"]);
52
52
 
53
+ const codexAuth = process.env.SELFBENCH_CODEX_AUTH_JSON;
54
+ const apiKey = process.env.OPENAI_API_KEY;
55
+ if (!apiKey && !codexAuth) {
56
+ fail("OPENAI_API_KEY or SELFBENCH_CODEX_AUTH_JSON is required");
57
+ }
53
58
  const codexHome = join(homedir(), ".codex");
54
59
  const authPath = join(codexHome, "auth.json");
55
60
  await mkdir(codexHome, { recursive: true });
56
- await writeFile(
57
- authPath,
58
- process.env.SELFBENCH_CODEX_AUTH_JSON ?? fail("SELFBENCH_CODEX_AUTH_JSON is required"),
59
- );
61
+ await writeFile(authPath, codexAuth ?? `${JSON.stringify({ OPENAI_API_KEY: apiKey })}\n`);
60
62
  await chmod(authPath, 0o600);
61
63
  const promptPath = join(tmpdir(), `selfbench-repair-${taskId}.md`);
62
64
  await writeFile(
@@ -86,7 +88,7 @@ const codex = await runCommand(
86
88
  {
87
89
  allowFailure: true,
88
90
  timeoutMs: 90 * 60 * 1000,
89
- env: withoutApiKey(process.env),
91
+ env: process.env,
90
92
  input: await readFile(promptPath, "utf8"),
91
93
  onOutput: (stream, chunk) => {
92
94
  (stream === "stdout" ? process.stdout : process.stderr).write(chunk);
@@ -149,12 +151,6 @@ await writeFile(
149
151
  )}\n`,
150
152
  );
151
153
 
152
- function withoutApiKey(environment: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
153
- const output = { ...environment };
154
- delete output.OPENAI_API_KEY;
155
- return output;
156
- }
157
-
158
154
  function fail(message: string): never {
159
155
  throw new Error(message);
160
156
  }
@@ -1,17 +1,19 @@
1
1
  import { readFile, writeFile } from "node:fs/promises";
2
2
  import { reviewCouplingWithCodex } from "./codex-review.js";
3
3
 
4
+ const apiKey = process.env.OPENAI_API_KEY;
4
5
  const authJson = process.env.SELFBENCH_PI_AUTH_JSON;
5
6
  const outputPath = process.env.SELFBENCH_REVIEW_OUTPUT;
6
- if (!authJson) {
7
- throw new Error("SELFBENCH_PI_AUTH_JSON is required");
7
+ if (!apiKey && !authJson) {
8
+ throw new Error("OPENAI_API_KEY or SELFBENCH_PI_AUTH_JSON is required");
8
9
  }
9
10
  if (!outputPath) {
10
11
  throw new Error("SELFBENCH_REVIEW_OUTPUT is required");
11
12
  }
12
13
 
13
14
  const review = await reviewCouplingWithCodex({
14
- authJson,
15
+ ...(apiKey ? { apiKey } : {}),
16
+ ...(authJson ? { authJson } : {}),
15
17
  prompt: await readFile("/work/review-input.md", "utf8"),
16
18
  });
17
19
  await writeFile(outputPath, `${JSON.stringify(review, null, 2)}\n`, { flag: "wx" });
@@ -68,10 +68,19 @@ await Promise.all([
68
68
  ]);
69
69
 
70
70
  const piHome = join(homedir(), ".pi/agent");
71
- const piAuth = process.env.SELFBENCH_PI_AUTH_JSON ?? fail("SELFBENCH_PI_AUTH_JSON is required");
71
+ const piAuth = process.env.SELFBENCH_PI_AUTH_JSON;
72
+ if (!process.env.OPENAI_API_KEY && !piAuth) {
73
+ fail("OPENAI_API_KEY or SELFBENCH_PI_AUTH_JSON is required");
74
+ }
72
75
  await mkdir(piHome, { recursive: true });
73
76
  await Promise.all([
74
- writeFile(join(piHome, "auth.json"), piAuth).then(() => chmod(join(piHome, "auth.json"), 0o600)),
77
+ ...(piAuth
78
+ ? [
79
+ writeFile(join(piHome, "auth.json"), piAuth).then(() =>
80
+ chmod(join(piHome, "auth.json"), 0o600),
81
+ ),
82
+ ]
83
+ : []),
75
84
  writeFile(join(piHome, "settings.json"), `${JSON.stringify({ transport: "auto" })}\n`),
76
85
  ]);
77
86
  const promptPath = join(tmpdir(), `selfbench-validation-repair-${originalDefinition.taskId}.md`);
@@ -98,7 +107,7 @@ const pi = await runCommand(
98
107
  "--no-context-files",
99
108
  "--no-extensions",
100
109
  "--provider",
101
- "openai-codex",
110
+ process.env.OPENAI_API_KEY ? "openai" : "openai-codex",
102
111
  "--model",
103
112
  process.env.SELFBENCH_REPAIR_MODEL ?? "gpt-5.6-sol",
104
113
  "--thinking",
@@ -111,7 +120,7 @@ const pi = await runCommand(
111
120
  allowFailure: true,
112
121
  timeoutMs: 90 * 60 * 1000,
113
122
  cwd: repositoryDirectory,
114
- env: withoutApiKey(process.env),
123
+ env: process.env,
115
124
  onOutput: (stream, chunk) => {
116
125
  (stream === "stdout" ? process.stdout : process.stderr).write(chunk);
117
126
  },
@@ -163,12 +172,6 @@ await Promise.all([
163
172
  ),
164
173
  ]);
165
174
 
166
- function withoutApiKey(environment: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
167
- const output = { ...environment };
168
- delete output.OPENAI_API_KEY;
169
- return output;
170
- }
171
-
172
175
  function fail(message: string): never {
173
176
  throw new Error(message);
174
177
  }
@@ -3,6 +3,34 @@ import { homedir } from "node:os";
3
3
  import { join } from "node:path";
4
4
  import { runCommand } from "./process.js";
5
5
 
6
+ export interface PiModelAuth {
7
+ readonly provider: "openai" | "openai-codex";
8
+ readonly apiKey?: string;
9
+ readonly authJson?: string;
10
+ }
11
+
12
+ export interface CodexModelAuth {
13
+ readonly apiKey?: string;
14
+ readonly authJson?: string;
15
+ }
16
+
17
+ export function openAiApiKey(): string | undefined {
18
+ const key = process.env.OPENAI_API_KEY?.trim();
19
+ return key || undefined;
20
+ }
21
+
22
+ export async function loadPiModelAuth(): Promise<PiModelAuth> {
23
+ const apiKey = openAiApiKey();
24
+ return apiKey
25
+ ? { provider: "openai", apiKey }
26
+ : { provider: "openai-codex", authJson: await loadPiSubscriptionAuth() };
27
+ }
28
+
29
+ export async function loadCodexModelAuth(): Promise<CodexModelAuth> {
30
+ const apiKey = openAiApiKey();
31
+ return apiKey ? { apiKey } : { authJson: await loadCodexSubscriptionAuth() };
32
+ }
33
+
6
34
  export async function loadPiSubscriptionAuth(): Promise<string> {
7
35
  const raw =
8
36
  process.env.SELFBENCH_PI_AUTH_JSON ??
@@ -28,7 +56,8 @@ export function assertCodexSubscriptionAuth(value: unknown, source = "Codex auth
28
56
 
29
57
  export async function loadCodexSubscriptionAuth(): Promise<string> {
30
58
  const path = process.env.CODEX_AUTH_JSON_PATH ?? join(homedir(), ".codex/auth.json");
31
- const parsed = JSON.parse(await readFile(path, "utf8")) as unknown;
59
+ const raw = process.env.SELFBENCH_CODEX_AUTH_JSON ?? (await readFile(path, "utf8"));
60
+ const parsed = JSON.parse(raw) as unknown;
32
61
  assertCodexSubscriptionAuth(parsed, path);
33
62
  const auth = parsed as Record<string, unknown>;
34
63
  return JSON.stringify({
@@ -38,22 +67,6 @@ export async function loadCodexSubscriptionAuth(): Promise<string> {
38
67
  });
39
68
  }
40
69
 
41
- export async function subscriptionBearerToken(model: string): Promise<string> {
42
- const result = await runCommand("pi", [
43
- "auth",
44
- "print-bearer-token",
45
- "--provider",
46
- "openai-codex",
47
- "--model",
48
- model,
49
- ]);
50
- const token = result.stdout.trim();
51
- if (!token) {
52
- throw new Error(`Pi returned no openai-codex subscription bearer token for ${model}`);
53
- }
54
- return token;
55
- }
56
-
57
70
  export async function githubToken(): Promise<string | undefined> {
58
71
  if (process.env.GH_TOKEN) {
59
72
  return process.env.GH_TOKEN;