self-bench 0.3.2 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +0 -1
- package/Dockerfile +2 -1
- package/README.md +101 -177
- package/bun.lock +12 -12
- package/compose.yaml +4 -4
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +40 -2
- package/dist/api.js.map +1 -1
- package/dist/artifacts.d.ts +0 -1
- package/dist/artifacts.d.ts.map +1 -1
- package/dist/artifacts.js +0 -3
- package/dist/artifacts.js.map +1 -1
- package/dist/build-metadata.js +1 -1
- package/dist/cli.js +8 -0
- package/dist/cli.js.map +1 -1
- package/dist/codex-review.d.ts +2 -1
- package/dist/codex-review.d.ts.map +1 -1
- package/dist/codex-review.js +26 -22
- package/dist/codex-review.js.map +1 -1
- package/dist/config.d.ts +0 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +0 -2
- package/dist/config.js.map +1 -1
- package/dist/contracts.d.ts +1 -1
- package/dist/contracts.d.ts.map +1 -1
- package/dist/eval-main.js +2 -2
- package/dist/evaluate.d.ts.map +1 -1
- package/dist/evaluate.js +12 -7
- package/dist/evaluate.js.map +1 -1
- package/dist/provenance.d.ts +0 -1
- package/dist/provenance.d.ts.map +1 -1
- package/dist/provenance.js.map +1 -1
- package/dist/reaudit-main.js +9 -6
- package/dist/reaudit-main.js.map +1 -1
- package/dist/repair-main.js +7 -4
- package/dist/repair-main.js.map +1 -1
- package/dist/review/assets/index-CZ3G759O.js +1568 -0
- package/dist/review/assets/index-DgSLzX8l.css +1 -0
- package/dist/review/assets/{mojo-Blby8dQK.js → mojo-DJz3ZmWd.js} +1 -1
- package/dist/review/assets/typst-BUadGCkm.js +1 -0
- package/dist/review/index.html +2 -2
- package/dist/sandbox-repair.bundle.js +8 -7
- package/dist/sandbox-repair.js +7 -7
- package/dist/sandbox-repair.js.map +1 -1
- package/dist/sandbox-review.bundle.js +29 -25
- package/dist/sandbox-review.js +5 -3
- package/dist/sandbox-review.js.map +1 -1
- package/dist/sandbox-validation-repair.bundle.js +9 -9
- package/dist/sandbox-validation-repair.js +11 -9
- package/dist/sandbox-validation-repair.js.map +1 -1
- package/dist/subscription-auth.d.ts +12 -1
- package/dist/subscription-auth.d.ts.map +1 -1
- package/dist/subscription-auth.js +16 -16
- package/dist/subscription-auth.js.map +1 -1
- package/dist/{activities.d.ts → temporal/activities.d.ts} +2 -3
- package/dist/temporal/activities.d.ts.map +1 -0
- package/dist/{activities.js → temporal/activities.js} +44 -33
- package/dist/temporal/activities.js.map +1 -0
- package/dist/{temporal.d.ts → temporal/connection.d.ts} +2 -2
- package/dist/temporal/connection.d.ts.map +1 -0
- package/dist/{temporal.js → temporal/connection.js} +1 -1
- package/dist/temporal/connection.js.map +1 -0
- package/dist/{worker-main.d.ts.map → temporal/worker-main.d.ts.map} +1 -1
- package/dist/{worker-main.js → temporal/worker-main.js} +5 -9
- package/dist/temporal/worker-main.js.map +1 -0
- package/dist/{workflow.d.ts → temporal/workflow.d.ts} +1 -1
- package/dist/temporal/workflow.d.ts.map +1 -0
- package/dist/temporal/workflow.js.map +1 -0
- package/docs/evaluations.md +13 -12
- package/docs/operations.md +10 -19
- package/package.json +6 -6
- package/src/api.ts +40 -2
- package/src/artifacts.ts +0 -4
- package/src/cli.ts +9 -0
- package/src/codex-review.ts +63 -54
- package/src/config.ts +0 -3
- package/src/contracts.ts +0 -5
- package/src/eval-main.ts +2 -2
- package/src/evaluate.ts +13 -8
- package/src/provenance.ts +0 -1
- package/src/reaudit-main.ts +9 -6
- package/src/repair-main.ts +7 -4
- package/src/sandbox-repair.ts +7 -11
- package/src/sandbox-review.ts +5 -3
- package/src/sandbox-validation-repair.ts +13 -10
- package/src/subscription-auth.ts +30 -17
- package/src/{activities.ts → temporal/activities.ts} +45 -40
- package/src/{temporal.ts → temporal/connection.ts} +1 -1
- package/src/{worker-main.ts → temporal/worker-main.ts} +5 -9
- package/src/{workflow.ts → temporal/workflow.ts} +10 -10
- package/dist/activities.d.ts.map +0 -1
- package/dist/activities.js.map +0 -1
- package/dist/extensions/review.d.ts +0 -3
- package/dist/extensions/review.d.ts.map +0 -1
- package/dist/extensions/review.js +0 -44
- package/dist/extensions/review.js.map +0 -1
- package/dist/review/assets/index-Dq-6DFzI.css +0 -1
- package/dist/review/assets/index-DurW_RD7.js +0 -1567
- package/dist/review/assets/typst-DHCkPAjA.js +0 -1
- package/dist/temporal.d.ts.map +0 -1
- package/dist/temporal.js.map +0 -1
- package/dist/worker-main.js.map +0 -1
- package/dist/workflow.d.ts.map +0 -1
- package/dist/workflow.js.map +0 -1
- package/src/extensions/review.ts +0 -54
- /package/dist/{worker-main.d.ts → temporal/worker-main.d.ts} +0 -0
- /package/dist/{workflow.js → temporal/workflow.js} +0 -0
package/src/artifacts.ts
CHANGED
|
@@ -355,7 +355,3 @@ function verifyArtifact(reference: ArtifactRef, value: Uint8Array): void {
|
|
|
355
355
|
throw new Error(`artifact integrity check failed: ${reference.uri}`);
|
|
356
356
|
}
|
|
357
357
|
}
|
|
358
|
-
|
|
359
|
-
export async function artifactSize(path: string): Promise<number> {
|
|
360
|
-
return (await stat(path)).size;
|
|
361
|
-
}
|
package/src/cli.ts
CHANGED
|
@@ -22,6 +22,9 @@ switch (command) {
|
|
|
22
22
|
case "run":
|
|
23
23
|
await run(rest);
|
|
24
24
|
break;
|
|
25
|
+
case "down":
|
|
26
|
+
await down();
|
|
27
|
+
break;
|
|
25
28
|
case "status":
|
|
26
29
|
await passthrough("GET", `/v1/runs/${requiredArgument(rest, "run ID")}`);
|
|
27
30
|
break;
|
|
@@ -97,6 +100,11 @@ async function up(args: string[]): Promise<void> {
|
|
|
97
100
|
console.log(`SelfBench is running with the ${backend} backend at http://127.0.0.1:8080`);
|
|
98
101
|
}
|
|
99
102
|
|
|
103
|
+
async function down(): Promise<void> {
|
|
104
|
+
const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
105
|
+
await runCommand("docker", ["compose", "--file", resolve(projectRoot, "compose.yaml"), "down"]);
|
|
106
|
+
}
|
|
107
|
+
|
|
100
108
|
async function run(args: string[]): Promise<void> {
|
|
101
109
|
const parsed = parseArgs({
|
|
102
110
|
args,
|
|
@@ -333,6 +341,7 @@ function printHelp(): void {
|
|
|
333
341
|
|
|
334
342
|
Usage:
|
|
335
343
|
self-bench up [--backend docker|modal] [--modal-config PATH]
|
|
344
|
+
self-bench down
|
|
336
345
|
self-bench run --repo PATH [--easy-count N] [--medium-count N] [--hard-count N]
|
|
337
346
|
[--model MODEL] [--run-id ID] [--wait] [--output OUTPUT.tar.gz]
|
|
338
347
|
self-bench status RUN_ID
|
package/src/codex-review.ts
CHANGED
|
@@ -28,45 +28,54 @@ export const couplingReviewSchema = z.object({
|
|
|
28
28
|
export type CouplingReview = z.infer<typeof couplingReviewSchema>;
|
|
29
29
|
|
|
30
30
|
export async function reviewCouplingWithCodex(input: {
|
|
31
|
-
readonly
|
|
31
|
+
readonly apiKey?: string;
|
|
32
|
+
readonly authJson?: string;
|
|
32
33
|
readonly prompt: string;
|
|
33
34
|
readonly signal?: AbortSignal;
|
|
34
35
|
readonly fetch?: typeof fetch;
|
|
35
36
|
}): Promise<CouplingReview> {
|
|
36
|
-
const
|
|
37
|
-
|
|
37
|
+
const subscription =
|
|
38
|
+
!input.apiKey && input.authJson ? parseCredential(input.authJson) : undefined;
|
|
39
|
+
if (!input.apiKey && !subscription) {
|
|
40
|
+
throw new Error("OpenAI model authentication is required");
|
|
41
|
+
}
|
|
38
42
|
const request = input.fetch ?? fetch;
|
|
39
|
-
const response = await request(
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
43
|
+
const response = await request(
|
|
44
|
+
subscription
|
|
45
|
+
? "https://chatgpt.com/backend-api/codex/responses"
|
|
46
|
+
: "https://api.openai.com/v1/responses",
|
|
47
|
+
{
|
|
48
|
+
method: "POST",
|
|
49
|
+
headers: {
|
|
50
|
+
Authorization: `Bearer ${subscription?.access ?? input.apiKey}`,
|
|
51
|
+
...(subscription ? { "ChatGPT-Account-Id": accountIdFromToken(subscription.access) } : {}),
|
|
52
|
+
Originator: "selfbench",
|
|
53
|
+
"OpenAI-Beta": "responses=experimental",
|
|
54
|
+
Accept: "text/event-stream",
|
|
55
|
+
"Content-Type": "application/json",
|
|
56
|
+
},
|
|
57
|
+
body: JSON.stringify({
|
|
58
|
+
model: COUPLING_REVIEW_MODEL,
|
|
59
|
+
store: false,
|
|
60
|
+
stream: true,
|
|
61
|
+
instructions:
|
|
62
|
+
"You are an independent benchmark-quality reviewer. Analyze the supplied evidence and call submit_review exactly once.",
|
|
63
|
+
input: [
|
|
64
|
+
{
|
|
65
|
+
role: "user",
|
|
66
|
+
content: [{ type: "input_text", text: input.prompt }],
|
|
67
|
+
},
|
|
68
|
+
],
|
|
69
|
+
reasoning: { effort: "high", summary: "auto" },
|
|
70
|
+
text: { verbosity: "low" },
|
|
71
|
+
include: ["reasoning.encrypted_content"],
|
|
72
|
+
tools: [reviewTool()],
|
|
73
|
+
tool_choice: { type: "function", name: "submit_review" },
|
|
74
|
+
parallel_tool_calls: false,
|
|
75
|
+
}),
|
|
76
|
+
...(input.signal ? { signal: input.signal } : {}),
|
|
48
77
|
},
|
|
49
|
-
|
|
50
|
-
model: COUPLING_REVIEW_MODEL,
|
|
51
|
-
store: false,
|
|
52
|
-
stream: true,
|
|
53
|
-
instructions:
|
|
54
|
-
"You are an independent benchmark-quality reviewer. Analyze the supplied evidence and call submit_review exactly once.",
|
|
55
|
-
input: [
|
|
56
|
-
{
|
|
57
|
-
role: "user",
|
|
58
|
-
content: [{ type: "input_text", text: input.prompt }],
|
|
59
|
-
},
|
|
60
|
-
],
|
|
61
|
-
reasoning: { effort: "high", summary: "auto" },
|
|
62
|
-
text: { verbosity: "low" },
|
|
63
|
-
include: ["reasoning.encrypted_content"],
|
|
64
|
-
tools: [reviewTool()],
|
|
65
|
-
tool_choice: { type: "function", name: "submit_review" },
|
|
66
|
-
parallel_tool_calls: false,
|
|
67
|
-
}),
|
|
68
|
-
...(input.signal ? { signal: input.signal } : {}),
|
|
69
|
-
});
|
|
78
|
+
);
|
|
70
79
|
const responseText = await response.text();
|
|
71
80
|
if (!response.ok) {
|
|
72
81
|
throw new Error(
|
|
@@ -76,6 +85,27 @@ export async function reviewCouplingWithCodex(input: {
|
|
|
76
85
|
return parseCodexReviewEvents(responseText);
|
|
77
86
|
}
|
|
78
87
|
|
|
88
|
+
function parseCredential(value: string): { access: string } {
|
|
89
|
+
const parsed = JSON.parse(value) as unknown;
|
|
90
|
+
const credential = isRecord(parsed) ? parsed["openai-codex"] : undefined;
|
|
91
|
+
if (!isRecord(credential) || typeof credential.access !== "string") {
|
|
92
|
+
throw new Error("Pi auth does not contain an OpenAI Codex subscription access token");
|
|
93
|
+
}
|
|
94
|
+
return { access: credential.access };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function accountIdFromToken(token: string): string {
|
|
98
|
+
const payload = JSON.parse(
|
|
99
|
+
Buffer.from(token.split(".")[1] ?? "", "base64url").toString("utf8"),
|
|
100
|
+
) as unknown;
|
|
101
|
+
const auth = isRecord(payload) ? payload["https://api.openai.com/auth"] : undefined;
|
|
102
|
+
const accountId = isRecord(auth) ? auth.chatgpt_account_id : undefined;
|
|
103
|
+
if (typeof accountId !== "string" || !accountId) {
|
|
104
|
+
throw new Error("OpenAI Codex subscription token has no ChatGPT account ID");
|
|
105
|
+
}
|
|
106
|
+
return accountId;
|
|
107
|
+
}
|
|
108
|
+
|
|
79
109
|
export function parseCodexReviewEvents(value: string): CouplingReview {
|
|
80
110
|
for (const line of value.split("\n")) {
|
|
81
111
|
if (!line.startsWith("data: ")) {
|
|
@@ -148,27 +178,6 @@ ${goldPatch}
|
|
|
148
178
|
`;
|
|
149
179
|
}
|
|
150
180
|
|
|
151
|
-
function parseCredential(value: string): { access: string } {
|
|
152
|
-
const parsed = JSON.parse(value) as unknown;
|
|
153
|
-
const credential = isRecord(parsed) ? parsed["openai-codex"] : undefined;
|
|
154
|
-
if (!isRecord(credential) || typeof credential.access !== "string") {
|
|
155
|
-
throw new Error("Pi auth does not contain an OpenAI Codex subscription access token");
|
|
156
|
-
}
|
|
157
|
-
return { access: credential.access };
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
function accountIdFromToken(token: string): string {
|
|
161
|
-
const payload = JSON.parse(
|
|
162
|
-
Buffer.from(token.split(".")[1] ?? "", "base64url").toString("utf8"),
|
|
163
|
-
) as unknown;
|
|
164
|
-
const auth = isRecord(payload) ? payload["https://api.openai.com/auth"] : undefined;
|
|
165
|
-
const accountId = isRecord(auth) ? auth.chatgpt_account_id : undefined;
|
|
166
|
-
if (typeof accountId !== "string" || !accountId) {
|
|
167
|
-
throw new Error("OpenAI Codex subscription token has no ChatGPT account ID");
|
|
168
|
-
}
|
|
169
|
-
return accountId;
|
|
170
|
-
}
|
|
171
|
-
|
|
172
181
|
function reviewTool(): Record<string, unknown> {
|
|
173
182
|
return {
|
|
174
183
|
type: "function",
|
package/src/config.ts
CHANGED
|
@@ -3,7 +3,6 @@ import { z } from "zod";
|
|
|
3
3
|
const environmentSchema = z.object({
|
|
4
4
|
SELFBENCH_API_HOST: z.string().default("127.0.0.1"),
|
|
5
5
|
SELFBENCH_API_PORT: z.coerce.number().int().min(1).max(65535).default(8080),
|
|
6
|
-
SELFBENCH_API_URL: z.string().url().default("http://127.0.0.1:8080"),
|
|
7
6
|
SELFBENCH_API_TOKEN: z.string().min(1).optional(),
|
|
8
7
|
SELFBENCH_ARTIFACT_BACKEND: z.enum(["local", "gcs"]).default("local"),
|
|
9
8
|
SELFBENCH_ARTIFACT_DIR: z.string().default(".selfbench/artifacts"),
|
|
@@ -36,7 +35,6 @@ const environmentSchema = z.object({
|
|
|
36
35
|
export interface SelfBenchConfig {
|
|
37
36
|
readonly apiHost: string;
|
|
38
37
|
readonly apiPort: number;
|
|
39
|
-
readonly apiUrl: string;
|
|
40
38
|
readonly apiToken?: string;
|
|
41
39
|
readonly buildCommit?: string;
|
|
42
40
|
readonly activityConcurrency: number;
|
|
@@ -81,7 +79,6 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBe
|
|
|
81
79
|
return {
|
|
82
80
|
apiHost: value.SELFBENCH_API_HOST,
|
|
83
81
|
apiPort: value.SELFBENCH_API_PORT,
|
|
84
|
-
apiUrl: value.SELFBENCH_API_URL,
|
|
85
82
|
...(value.SELFBENCH_API_TOKEN ? { apiToken: value.SELFBENCH_API_TOKEN } : {}),
|
|
86
83
|
...(value.SELFBENCH_BUILD_COMMIT
|
|
87
84
|
? { buildCommit: value.SELFBENCH_BUILD_COMMIT.toLowerCase() }
|
package/src/contracts.ts
CHANGED
|
@@ -130,15 +130,10 @@ export type RunPhase =
|
|
|
130
130
|
| "queued"
|
|
131
131
|
| "discovering"
|
|
132
132
|
| "authoring"
|
|
133
|
-
| "validating"
|
|
134
|
-
| "reviewing"
|
|
135
|
-
| "repairing"
|
|
136
|
-
| "auditing"
|
|
137
133
|
| "exporting"
|
|
138
134
|
| "complete"
|
|
139
135
|
| "blocked"
|
|
140
136
|
| "failed"
|
|
141
|
-
| "cancelling"
|
|
142
137
|
| "cancelled";
|
|
143
138
|
|
|
144
139
|
export interface TaskProgress {
|
package/src/eval-main.ts
CHANGED
|
@@ -18,7 +18,7 @@ const parsed = parseArgs({
|
|
|
18
18
|
strict: true,
|
|
19
19
|
});
|
|
20
20
|
if (parsed.values.help) {
|
|
21
|
-
console.log(`Run Harbor tasks through the fixed Codex
|
|
21
|
+
console.log(`Run Harbor tasks through the fixed Codex model matrix.
|
|
22
22
|
|
|
23
23
|
Usage:
|
|
24
24
|
self-bench-eval (--export FILE.tar.gz | --tasks DIRECTORY) --jobs DIRECTORY [options]
|
|
@@ -29,7 +29,7 @@ Options:
|
|
|
29
29
|
--environment docker|modal Execution environment (default: modal)
|
|
30
30
|
--concurrency N Concurrent trials (default: 3)
|
|
31
31
|
--model MODEL Run only this model; may be repeated
|
|
32
|
-
--auth FILE Codex ChatGPT auth.json
|
|
32
|
+
--auth FILE Optional Codex ChatGPT auth.json fallback
|
|
33
33
|
-h, --help Show this help`);
|
|
34
34
|
process.exit(0);
|
|
35
35
|
}
|
package/src/evaluate.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { basename, join, resolve } from "node:path";
|
|
|
4
4
|
import { archiveIncompleteHarborJob, tryReadHarborJobResult } from "./harbor-results.js";
|
|
5
5
|
import { parallelMap } from "./parallel.js";
|
|
6
6
|
import { runCommand } from "./process.js";
|
|
7
|
-
import { assertCodexSubscriptionAuth } from "./subscription-auth.js";
|
|
7
|
+
import { assertCodexSubscriptionAuth, openAiApiKey } from "./subscription-auth.js";
|
|
8
8
|
|
|
9
9
|
export const MATRIX_MODELS = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"] as const;
|
|
10
10
|
export type MatrixModel = (typeof MATRIX_MODELS)[number];
|
|
@@ -40,8 +40,11 @@ export async function runMatrix(options: MatrixOptions): Promise<readonly Matrix
|
|
|
40
40
|
const tasksDirectory = join(root, "tasks");
|
|
41
41
|
await mkdir(root, { recursive: true });
|
|
42
42
|
const taskDirectories = await resolveMatrixTasks(options, tasksDirectory);
|
|
43
|
+
const apiKey = openAiApiKey();
|
|
43
44
|
const authPath = resolve(options.authPath ?? join(homedir(), ".codex/auth.json"));
|
|
44
|
-
|
|
45
|
+
if (!apiKey) {
|
|
46
|
+
await assertSubscriptionAuth(authPath);
|
|
47
|
+
}
|
|
45
48
|
const models = options.models ?? MATRIX_MODELS;
|
|
46
49
|
if (models.length < 1) {
|
|
47
50
|
throw new Error("provide at least one evaluation model");
|
|
@@ -55,7 +58,7 @@ export async function runMatrix(options: MatrixOptions): Promise<readonly Matrix
|
|
|
55
58
|
jobsDirectory: root,
|
|
56
59
|
harborPath: options.harborPath ?? "harbor",
|
|
57
60
|
environment: options.environment ?? "modal",
|
|
58
|
-
authPath,
|
|
61
|
+
...(!apiKey ? { authPath } : {}),
|
|
59
62
|
...item,
|
|
60
63
|
});
|
|
61
64
|
completed += 1;
|
|
@@ -69,7 +72,7 @@ export async function runMatrix(options: MatrixOptions): Promise<readonly Matrix
|
|
|
69
72
|
schemaVersion: 1,
|
|
70
73
|
agent: "codex",
|
|
71
74
|
agentVersion: CODEX_VERSION,
|
|
72
|
-
auth: "codex-subscription",
|
|
75
|
+
auth: apiKey ? "openai-api-key" : "codex-subscription",
|
|
73
76
|
reasoningEffort: "high",
|
|
74
77
|
models,
|
|
75
78
|
taskCount: taskDirectories.length,
|
|
@@ -112,7 +115,7 @@ async function runTrial(input: {
|
|
|
112
115
|
readonly jobsDirectory: string;
|
|
113
116
|
readonly harborPath: string;
|
|
114
117
|
readonly environment: "docker" | "modal";
|
|
115
|
-
readonly authPath
|
|
118
|
+
readonly authPath?: string;
|
|
116
119
|
}): Promise<MatrixTrialSummary> {
|
|
117
120
|
const taskId = basename(input.taskDirectory);
|
|
118
121
|
const jobName = `${taskId}-${input.model}`.toLowerCase().replace(/[^a-z0-9_.-]/g, "-");
|
|
@@ -123,9 +126,11 @@ async function runTrial(input: {
|
|
|
123
126
|
await archiveIncompleteHarborJob(input.jobsDirectory, jobName);
|
|
124
127
|
|
|
125
128
|
const environment = { ...process.env };
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
+
if (input.authPath) {
|
|
130
|
+
delete environment.OPENAI_API_KEY;
|
|
131
|
+
environment.CODEX_FORCE_AUTH_JSON = "1";
|
|
132
|
+
environment.CODEX_AUTH_JSON_PATH = input.authPath;
|
|
133
|
+
}
|
|
129
134
|
let result: Awaited<ReturnType<typeof runCommand>>;
|
|
130
135
|
try {
|
|
131
136
|
result = await runCommand(
|
package/src/provenance.ts
CHANGED
|
@@ -4,7 +4,6 @@ import { assertPullRequestBelongsToRepository, githubRepository } from "./github
|
|
|
4
4
|
import { runCommand } from "./process.js";
|
|
5
5
|
|
|
6
6
|
export type SessionProvenanceFormat = "codex" | "claude-code" | "pi" | "generic";
|
|
7
|
-
export type ProvenanceFormat = SessionProvenanceFormat | "github-pull-request";
|
|
8
7
|
|
|
9
8
|
interface ProvenanceMessageBase {
|
|
10
9
|
readonly sessionId: string;
|
package/src/reaudit-main.ts
CHANGED
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
import { parallelMap } from "./parallel.js";
|
|
21
21
|
import { runCommand } from "./process.js";
|
|
22
22
|
import { createSandboxExecutor, type SandboxExecutor } from "./sandbox.js";
|
|
23
|
-
import {
|
|
23
|
+
import { loadPiModelAuth } from "./subscription-auth.js";
|
|
24
24
|
|
|
25
25
|
const parsed = parseArgs({
|
|
26
26
|
options: {
|
|
@@ -50,9 +50,9 @@ const outputPath = resolve(parsed.values.output ?? fail("--output is required"))
|
|
|
50
50
|
const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
|
|
51
51
|
const config = loadConfig();
|
|
52
52
|
const sandbox = createSandboxExecutor(config.execution);
|
|
53
|
-
const [taskIds,
|
|
53
|
+
const [taskIds, modelAuth, reviewer] = await Promise.all([
|
|
54
54
|
readdir(tasksDirectory),
|
|
55
|
-
|
|
55
|
+
loadPiModelAuth(),
|
|
56
56
|
readFile(join(assetRoot(), "dist/sandbox-review.bundle.js")),
|
|
57
57
|
]);
|
|
58
58
|
const directories = (
|
|
@@ -79,7 +79,7 @@ const tasks = await parallelMap(directories, concurrency, async (taskDirectory)
|
|
|
79
79
|
const taskId = basename(taskDirectory);
|
|
80
80
|
console.error(`reviewing ${taskId}`);
|
|
81
81
|
try {
|
|
82
|
-
const report = await auditTask(taskDirectory, taskId,
|
|
82
|
+
const report = await auditTask(taskDirectory, taskId, modelAuth, reviewer, sandbox);
|
|
83
83
|
console.error(`${taskId}: ${report.resolution.verdict}`);
|
|
84
84
|
return report;
|
|
85
85
|
} catch (error) {
|
|
@@ -113,7 +113,7 @@ console.log(JSON.stringify({ output: outputPath, ...report.summary }, null, 2));
|
|
|
113
113
|
async function auditTask(
|
|
114
114
|
taskDirectory: string,
|
|
115
115
|
taskId: string,
|
|
116
|
-
|
|
116
|
+
modelAuth: { apiKey?: string; authJson?: string },
|
|
117
117
|
reviewer: Uint8Array,
|
|
118
118
|
sandbox: SandboxExecutor,
|
|
119
119
|
) {
|
|
@@ -151,7 +151,10 @@ async function auditTask(
|
|
|
151
151
|
},
|
|
152
152
|
],
|
|
153
153
|
outputPaths: ["/work/review.json"],
|
|
154
|
-
secrets: {
|
|
154
|
+
secrets: {
|
|
155
|
+
...(modelAuth.apiKey ? { OPENAI_API_KEY: modelAuth.apiKey } : {}),
|
|
156
|
+
...(modelAuth.authJson ? { SELFBENCH_PI_AUTH_JSON: modelAuth.authJson } : {}),
|
|
157
|
+
},
|
|
155
158
|
environment: { SELFBENCH_REVIEW_OUTPUT: "/work/review.json" },
|
|
156
159
|
command: ["node", "/work/sandbox-review.js"],
|
|
157
160
|
});
|
package/src/repair-main.ts
CHANGED
|
@@ -9,7 +9,7 @@ import { loadConfig } from "./config.js";
|
|
|
9
9
|
import { parallelMap } from "./parallel.js";
|
|
10
10
|
import { runCommand } from "./process.js";
|
|
11
11
|
import { createSandboxExecutor } from "./sandbox.js";
|
|
12
|
-
import {
|
|
12
|
+
import { loadCodexModelAuth } from "./subscription-auth.js";
|
|
13
13
|
|
|
14
14
|
const parsed = parseArgs({
|
|
15
15
|
options: {
|
|
@@ -79,8 +79,8 @@ for (const source of sourceTasks) {
|
|
|
79
79
|
|
|
80
80
|
const config = loadConfig();
|
|
81
81
|
const sandbox = createSandboxExecutor(config.execution);
|
|
82
|
-
const [
|
|
83
|
-
|
|
82
|
+
const [modelAuth, repairer] = await Promise.all([
|
|
83
|
+
loadCodexModelAuth(),
|
|
84
84
|
readFile(join(assetRoot(), "dist/sandbox-repair.bundle.js")),
|
|
85
85
|
]);
|
|
86
86
|
const results = await parallelMap([...coupled.values()], concurrency, async (taskReview) => {
|
|
@@ -103,7 +103,10 @@ const results = await parallelMap([...coupled.values()], concurrency, async (tas
|
|
|
103
103
|
{ path: "/work/sandbox-repair.js", contents: repairer },
|
|
104
104
|
],
|
|
105
105
|
outputPaths: ["/work/repaired-task.tar.gz", "/work/repair-report.json"],
|
|
106
|
-
secrets: {
|
|
106
|
+
secrets: {
|
|
107
|
+
...(modelAuth.apiKey ? { OPENAI_API_KEY: modelAuth.apiKey } : {}),
|
|
108
|
+
...(modelAuth.authJson ? { SELFBENCH_CODEX_AUTH_JSON: modelAuth.authJson } : {}),
|
|
109
|
+
},
|
|
107
110
|
environment: { SELFBENCH_REPAIR_MODEL: parsed.values.model ?? "gpt-5.6-sol" },
|
|
108
111
|
command: [
|
|
109
112
|
"node",
|
package/src/sandbox-repair.ts
CHANGED
|
@@ -50,13 +50,15 @@ await runCommand("git", [
|
|
|
50
50
|
]);
|
|
51
51
|
await runCommand("git", ["-C", repositoryDirectory, "add", "-N", "--all"]);
|
|
52
52
|
|
|
53
|
+
const codexAuth = process.env.SELFBENCH_CODEX_AUTH_JSON;
|
|
54
|
+
const apiKey = process.env.OPENAI_API_KEY;
|
|
55
|
+
if (!apiKey && !codexAuth) {
|
|
56
|
+
fail("OPENAI_API_KEY or SELFBENCH_CODEX_AUTH_JSON is required");
|
|
57
|
+
}
|
|
53
58
|
const codexHome = join(homedir(), ".codex");
|
|
54
59
|
const authPath = join(codexHome, "auth.json");
|
|
55
60
|
await mkdir(codexHome, { recursive: true });
|
|
56
|
-
await writeFile(
|
|
57
|
-
authPath,
|
|
58
|
-
process.env.SELFBENCH_CODEX_AUTH_JSON ?? fail("SELFBENCH_CODEX_AUTH_JSON is required"),
|
|
59
|
-
);
|
|
61
|
+
await writeFile(authPath, codexAuth ?? `${JSON.stringify({ OPENAI_API_KEY: apiKey })}\n`);
|
|
60
62
|
await chmod(authPath, 0o600);
|
|
61
63
|
const promptPath = join(tmpdir(), `selfbench-repair-${taskId}.md`);
|
|
62
64
|
await writeFile(
|
|
@@ -86,7 +88,7 @@ const codex = await runCommand(
|
|
|
86
88
|
{
|
|
87
89
|
allowFailure: true,
|
|
88
90
|
timeoutMs: 90 * 60 * 1000,
|
|
89
|
-
env:
|
|
91
|
+
env: process.env,
|
|
90
92
|
input: await readFile(promptPath, "utf8"),
|
|
91
93
|
onOutput: (stream, chunk) => {
|
|
92
94
|
(stream === "stdout" ? process.stdout : process.stderr).write(chunk);
|
|
@@ -149,12 +151,6 @@ await writeFile(
|
|
|
149
151
|
)}\n`,
|
|
150
152
|
);
|
|
151
153
|
|
|
152
|
-
function withoutApiKey(environment: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
|
|
153
|
-
const output = { ...environment };
|
|
154
|
-
delete output.OPENAI_API_KEY;
|
|
155
|
-
return output;
|
|
156
|
-
}
|
|
157
|
-
|
|
158
154
|
function fail(message: string): never {
|
|
159
155
|
throw new Error(message);
|
|
160
156
|
}
|
package/src/sandbox-review.ts
CHANGED
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
import { readFile, writeFile } from "node:fs/promises";
|
|
2
2
|
import { reviewCouplingWithCodex } from "./codex-review.js";
|
|
3
3
|
|
|
4
|
+
const apiKey = process.env.OPENAI_API_KEY;
|
|
4
5
|
const authJson = process.env.SELFBENCH_PI_AUTH_JSON;
|
|
5
6
|
const outputPath = process.env.SELFBENCH_REVIEW_OUTPUT;
|
|
6
|
-
if (!authJson) {
|
|
7
|
-
throw new Error("SELFBENCH_PI_AUTH_JSON is required");
|
|
7
|
+
if (!apiKey && !authJson) {
|
|
8
|
+
throw new Error("OPENAI_API_KEY or SELFBENCH_PI_AUTH_JSON is required");
|
|
8
9
|
}
|
|
9
10
|
if (!outputPath) {
|
|
10
11
|
throw new Error("SELFBENCH_REVIEW_OUTPUT is required");
|
|
11
12
|
}
|
|
12
13
|
|
|
13
14
|
const review = await reviewCouplingWithCodex({
|
|
14
|
-
|
|
15
|
+
...(apiKey ? { apiKey } : {}),
|
|
16
|
+
...(authJson ? { authJson } : {}),
|
|
15
17
|
prompt: await readFile("/work/review-input.md", "utf8"),
|
|
16
18
|
});
|
|
17
19
|
await writeFile(outputPath, `${JSON.stringify(review, null, 2)}\n`, { flag: "wx" });
|
|
@@ -68,10 +68,19 @@ await Promise.all([
|
|
|
68
68
|
]);
|
|
69
69
|
|
|
70
70
|
const piHome = join(homedir(), ".pi/agent");
|
|
71
|
-
const piAuth = process.env.SELFBENCH_PI_AUTH_JSON
|
|
71
|
+
const piAuth = process.env.SELFBENCH_PI_AUTH_JSON;
|
|
72
|
+
if (!process.env.OPENAI_API_KEY && !piAuth) {
|
|
73
|
+
fail("OPENAI_API_KEY or SELFBENCH_PI_AUTH_JSON is required");
|
|
74
|
+
}
|
|
72
75
|
await mkdir(piHome, { recursive: true });
|
|
73
76
|
await Promise.all([
|
|
74
|
-
|
|
77
|
+
...(piAuth
|
|
78
|
+
? [
|
|
79
|
+
writeFile(join(piHome, "auth.json"), piAuth).then(() =>
|
|
80
|
+
chmod(join(piHome, "auth.json"), 0o600),
|
|
81
|
+
),
|
|
82
|
+
]
|
|
83
|
+
: []),
|
|
75
84
|
writeFile(join(piHome, "settings.json"), `${JSON.stringify({ transport: "auto" })}\n`),
|
|
76
85
|
]);
|
|
77
86
|
const promptPath = join(tmpdir(), `selfbench-validation-repair-${originalDefinition.taskId}.md`);
|
|
@@ -98,7 +107,7 @@ const pi = await runCommand(
|
|
|
98
107
|
"--no-context-files",
|
|
99
108
|
"--no-extensions",
|
|
100
109
|
"--provider",
|
|
101
|
-
"openai-codex",
|
|
110
|
+
process.env.OPENAI_API_KEY ? "openai" : "openai-codex",
|
|
102
111
|
"--model",
|
|
103
112
|
process.env.SELFBENCH_REPAIR_MODEL ?? "gpt-5.6-sol",
|
|
104
113
|
"--thinking",
|
|
@@ -111,7 +120,7 @@ const pi = await runCommand(
|
|
|
111
120
|
allowFailure: true,
|
|
112
121
|
timeoutMs: 90 * 60 * 1000,
|
|
113
122
|
cwd: repositoryDirectory,
|
|
114
|
-
env:
|
|
123
|
+
env: process.env,
|
|
115
124
|
onOutput: (stream, chunk) => {
|
|
116
125
|
(stream === "stdout" ? process.stdout : process.stderr).write(chunk);
|
|
117
126
|
},
|
|
@@ -163,12 +172,6 @@ await Promise.all([
|
|
|
163
172
|
),
|
|
164
173
|
]);
|
|
165
174
|
|
|
166
|
-
function withoutApiKey(environment: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
|
|
167
|
-
const output = { ...environment };
|
|
168
|
-
delete output.OPENAI_API_KEY;
|
|
169
|
-
return output;
|
|
170
|
-
}
|
|
171
|
-
|
|
172
175
|
function fail(message: string): never {
|
|
173
176
|
throw new Error(message);
|
|
174
177
|
}
|
package/src/subscription-auth.ts
CHANGED
|
@@ -3,6 +3,34 @@ import { homedir } from "node:os";
|
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { runCommand } from "./process.js";
|
|
5
5
|
|
|
6
|
+
export interface PiModelAuth {
|
|
7
|
+
readonly provider: "openai" | "openai-codex";
|
|
8
|
+
readonly apiKey?: string;
|
|
9
|
+
readonly authJson?: string;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface CodexModelAuth {
|
|
13
|
+
readonly apiKey?: string;
|
|
14
|
+
readonly authJson?: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function openAiApiKey(): string | undefined {
|
|
18
|
+
const key = process.env.OPENAI_API_KEY?.trim();
|
|
19
|
+
return key || undefined;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export async function loadPiModelAuth(): Promise<PiModelAuth> {
|
|
23
|
+
const apiKey = openAiApiKey();
|
|
24
|
+
return apiKey
|
|
25
|
+
? { provider: "openai", apiKey }
|
|
26
|
+
: { provider: "openai-codex", authJson: await loadPiSubscriptionAuth() };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export async function loadCodexModelAuth(): Promise<CodexModelAuth> {
|
|
30
|
+
const apiKey = openAiApiKey();
|
|
31
|
+
return apiKey ? { apiKey } : { authJson: await loadCodexSubscriptionAuth() };
|
|
32
|
+
}
|
|
33
|
+
|
|
6
34
|
export async function loadPiSubscriptionAuth(): Promise<string> {
|
|
7
35
|
const raw =
|
|
8
36
|
process.env.SELFBENCH_PI_AUTH_JSON ??
|
|
@@ -28,7 +56,8 @@ export function assertCodexSubscriptionAuth(value: unknown, source = "Codex auth
|
|
|
28
56
|
|
|
29
57
|
export async function loadCodexSubscriptionAuth(): Promise<string> {
|
|
30
58
|
const path = process.env.CODEX_AUTH_JSON_PATH ?? join(homedir(), ".codex/auth.json");
|
|
31
|
-
const
|
|
59
|
+
const raw = process.env.SELFBENCH_CODEX_AUTH_JSON ?? (await readFile(path, "utf8"));
|
|
60
|
+
const parsed = JSON.parse(raw) as unknown;
|
|
32
61
|
assertCodexSubscriptionAuth(parsed, path);
|
|
33
62
|
const auth = parsed as Record<string, unknown>;
|
|
34
63
|
return JSON.stringify({
|
|
@@ -38,22 +67,6 @@ export async function loadCodexSubscriptionAuth(): Promise<string> {
|
|
|
38
67
|
});
|
|
39
68
|
}
|
|
40
69
|
|
|
41
|
-
export async function subscriptionBearerToken(model: string): Promise<string> {
|
|
42
|
-
const result = await runCommand("pi", [
|
|
43
|
-
"auth",
|
|
44
|
-
"print-bearer-token",
|
|
45
|
-
"--provider",
|
|
46
|
-
"openai-codex",
|
|
47
|
-
"--model",
|
|
48
|
-
model,
|
|
49
|
-
]);
|
|
50
|
-
const token = result.stdout.trim();
|
|
51
|
-
if (!token) {
|
|
52
|
-
throw new Error(`Pi returned no openai-codex subscription bearer token for ${model}`);
|
|
53
|
-
}
|
|
54
|
-
return token;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
70
|
export async function githubToken(): Promise<string | undefined> {
|
|
58
71
|
if (process.env.GH_TOKEN) {
|
|
59
72
|
return process.env.GH_TOKEN;
|