self-bench 0.3.3 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile.sandbox +1 -1
- package/README.md +20 -0
- package/bun.lock +48 -0
- package/compose.yaml +7 -1
- package/dist/agent-smoke-main.js +3 -2
- package/dist/agent-smoke-main.js.map +1 -1
- package/dist/agent-smoke.d.ts +2 -1
- package/dist/agent-smoke.d.ts.map +1 -1
- package/dist/agent-smoke.js +6 -1
- package/dist/agent-smoke.js.map +1 -1
- package/dist/api.d.ts +24 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +25 -18
- package/dist/api.js.map +1 -1
- package/dist/build-metadata.js +1 -1
- package/dist/cli.js +67 -11
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +36 -10
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +89 -15
- package/dist/config.js.map +1 -1
- package/dist/contracts.d.ts +22 -2
- package/dist/contracts.d.ts.map +1 -1
- package/dist/contracts.js +31 -6
- package/dist/contracts.js.map +1 -1
- package/dist/eval-main.js +3 -2
- package/dist/eval-main.js.map +1 -1
- package/dist/evaluate.d.ts +2 -1
- package/dist/evaluate.d.ts.map +1 -1
- package/dist/evaluate.js +2 -1
- package/dist/evaluate.js.map +1 -1
- package/dist/harbor-environment.d.ts +2 -0
- package/dist/harbor-environment.d.ts.map +1 -0
- package/dist/harbor-environment.js +15 -0
- package/dist/harbor-environment.js.map +1 -0
- package/dist/harbor-results.d.ts.map +1 -1
- package/dist/harbor-results.js +2 -0
- package/dist/harbor-results.js.map +1 -1
- package/dist/process.d.ts +2 -1
- package/dist/process.d.ts.map +1 -1
- package/dist/process.js.map +1 -1
- package/dist/providers.d.ts +8 -0
- package/dist/providers.d.ts.map +1 -0
- package/dist/providers.js +14 -0
- package/dist/providers.js.map +1 -0
- package/dist/reaudit-main.js +5 -5
- package/dist/reaudit-main.js.map +1 -1
- package/dist/repair-main.js +5 -5
- package/dist/repair-main.js.map +1 -1
- package/dist/sandbox-author.bundle.js +40 -6
- package/dist/sandbox-timeout.d.ts +11 -0
- package/dist/sandbox-timeout.d.ts.map +1 -0
- package/dist/sandbox-timeout.js +41 -0
- package/dist/sandbox-timeout.js.map +1 -0
- package/dist/sandbox-validation-repair.bundle.js +40 -6
- package/dist/sandbox.d.ts +2 -2
- package/dist/sandbox.d.ts.map +1 -1
- package/dist/sandbox.js +10 -3
- package/dist/sandbox.js.map +1 -1
- package/dist/temporal/activities.d.ts +2 -2
- package/dist/temporal/activities.d.ts.map +1 -1
- package/dist/temporal/activities.js +7 -1
- package/dist/temporal/activities.js.map +1 -1
- package/dist/temporal/worker-main.js +2 -2
- package/dist/temporal/worker-main.js.map +1 -1
- package/dist/terminal-prompts.d.ts +26 -0
- package/dist/terminal-prompts.d.ts.map +1 -0
- package/dist/terminal-prompts.js +137 -0
- package/dist/terminal-prompts.js.map +1 -0
- package/dist/terminal-reporter.d.ts +30 -0
- package/dist/terminal-reporter.d.ts.map +1 -0
- package/dist/terminal-reporter.js +90 -0
- package/dist/terminal-reporter.js.map +1 -0
- package/dist/validate-main.js +8 -2
- package/dist/validate-main.js.map +1 -1
- package/dist/vercel-cli.d.ts +64 -0
- package/dist/vercel-cli.d.ts.map +1 -0
- package/dist/vercel-cli.js +303 -0
- package/dist/vercel-cli.js.map +1 -0
- package/dist/vercel-command.d.ts +14 -0
- package/dist/vercel-command.d.ts.map +1 -0
- package/dist/vercel-command.js +93 -0
- package/dist/vercel-command.js.map +1 -0
- package/dist/vercel-executor.d.ts +14 -0
- package/dist/vercel-executor.d.ts.map +1 -0
- package/dist/vercel-executor.js +356 -0
- package/dist/vercel-executor.js.map +1 -0
- package/dist/vercel-fetch.d.ts +6 -0
- package/dist/vercel-fetch.d.ts.map +1 -0
- package/dist/vercel-fetch.js +39 -0
- package/dist/vercel-fetch.js.map +1 -0
- package/dist/vercel-profile.d.ts +34 -0
- package/dist/vercel-profile.d.ts.map +1 -0
- package/dist/vercel-profile.js +244 -0
- package/dist/vercel-profile.js.map +1 -0
- package/dist/vercel-runtime-image.d.ts +51 -0
- package/dist/vercel-runtime-image.d.ts.map +1 -0
- package/dist/vercel-runtime-image.js +123 -0
- package/dist/vercel-runtime-image.js.map +1 -0
- package/dist/vercel-setup-probe.d.ts +49 -0
- package/dist/vercel-setup-probe.d.ts.map +1 -0
- package/dist/vercel-setup-probe.js +151 -0
- package/dist/vercel-setup-probe.js.map +1 -0
- package/dist/vercel-setup.d.ts +42 -0
- package/dist/vercel-setup.d.ts.map +1 -0
- package/dist/vercel-setup.js +280 -0
- package/dist/vercel-setup.js.map +1 -0
- package/docs/evaluations.md +2 -0
- package/docs/operations.md +128 -6
- package/package.json +3 -1
- package/src/agent-smoke-main.ts +3 -2
- package/src/agent-smoke.ts +8 -2
- package/src/api.ts +31 -18
- package/src/cli.ts +72 -11
- package/src/config.ts +150 -25
- package/src/contracts.ts +33 -6
- package/src/eval-main.ts +3 -2
- package/src/evaluate.ts +5 -3
- package/src/harbor-environment.ts +17 -0
- package/src/harbor-results.ts +2 -0
- package/src/process.ts +3 -1
- package/src/providers.ts +22 -0
- package/src/reaudit-main.ts +8 -5
- package/src/repair-main.ts +8 -5
- package/src/sandbox-timeout.ts +56 -0
- package/src/sandbox.ts +15 -5
- package/src/temporal/activities.ts +9 -3
- package/src/temporal/worker-main.ts +2 -2
- package/src/terminal-prompts.ts +197 -0
- package/src/terminal-reporter.ts +126 -0
- package/src/validate-main.ts +9 -3
- package/src/vercel-cli.ts +403 -0
- package/src/vercel-command.ts +127 -0
- package/src/vercel-executor.ts +448 -0
- package/src/vercel-fetch.ts +45 -0
- package/src/vercel-profile.ts +319 -0
- package/src/vercel-runtime-image.ts +204 -0
- package/src/vercel-setup-probe.ts +229 -0
- package/src/vercel-setup.ts +419 -0
package/src/api.ts
CHANGED
|
@@ -11,6 +11,7 @@ import type { SelfBenchConfig } from "./config.js";
|
|
|
11
11
|
import {
|
|
12
12
|
artifactRefSchema,
|
|
13
13
|
type RunPhase,
|
|
14
|
+
type RunRequest,
|
|
14
15
|
type RunStatus,
|
|
15
16
|
repositoryRefSchema,
|
|
16
17
|
runRequestSchema,
|
|
@@ -68,26 +69,10 @@ export async function startApi(config: SelfBenchConfig): Promise<() => Promise<v
|
|
|
68
69
|
return;
|
|
69
70
|
}
|
|
70
71
|
if (request.method === "POST" && url.pathname === "/v1/runs") {
|
|
71
|
-
const
|
|
72
|
+
const workflowInput = buildRunRequest(
|
|
73
|
+
config,
|
|
72
74
|
JSON.parse((await readBody(request)).toString("utf8")),
|
|
73
75
|
);
|
|
74
|
-
const workflowInput = runRequestSchema.parse({
|
|
75
|
-
runId: submission.runId,
|
|
76
|
-
repository: submission.repository,
|
|
77
|
-
provenance: submission.provenance,
|
|
78
|
-
candidateCounts: submission.candidateCounts,
|
|
79
|
-
authoring: {
|
|
80
|
-
provider: "openai-codex",
|
|
81
|
-
model: submission.authoringModel,
|
|
82
|
-
reasoningEffort: "high",
|
|
83
|
-
},
|
|
84
|
-
version: {
|
|
85
|
-
selfbenchCommit: config.buildCommit ?? submission.selfbenchCommit,
|
|
86
|
-
executionBackend: config.execution.kind,
|
|
87
|
-
sandboxImage: config.execution.image,
|
|
88
|
-
schema: 1,
|
|
89
|
-
},
|
|
90
|
-
});
|
|
91
76
|
await client.workflow.start(selfBenchRunWorkflow, {
|
|
92
77
|
workflowId: workflowInput.runId,
|
|
93
78
|
taskQueue: config.temporal.taskQueue,
|
|
@@ -167,6 +152,34 @@ export async function startApi(config: SelfBenchConfig): Promise<() => Promise<v
|
|
|
167
152
|
};
|
|
168
153
|
}
|
|
169
154
|
|
|
155
|
+
export function buildRunRequest(
|
|
156
|
+
config: SelfBenchConfig,
|
|
157
|
+
submission: z.input<typeof submissionSchema>,
|
|
158
|
+
): RunRequest {
|
|
159
|
+
const parsed = submissionSchema.parse(submission);
|
|
160
|
+
return runRequestSchema.parse({
|
|
161
|
+
runId: parsed.runId,
|
|
162
|
+
repository: parsed.repository,
|
|
163
|
+
provenance: parsed.provenance,
|
|
164
|
+
candidateCounts: parsed.candidateCounts,
|
|
165
|
+
authoring: {
|
|
166
|
+
provider: "openai-codex",
|
|
167
|
+
model: parsed.authoringModel,
|
|
168
|
+
reasoningEffort: "high",
|
|
169
|
+
},
|
|
170
|
+
version: {
|
|
171
|
+
selfbenchCommit: config.buildCommit ?? parsed.selfbenchCommit,
|
|
172
|
+
executionBackend: config.execution.kind,
|
|
173
|
+
harborEnvironment: config.harborEnvironment,
|
|
174
|
+
sandboxImage: config.execution.image,
|
|
175
|
+
...(config.execution.kind === "vercel"
|
|
176
|
+
? { sandboxTimeoutCapMs: config.execution.timeoutCapMs }
|
|
177
|
+
: {}),
|
|
178
|
+
schema: 1,
|
|
179
|
+
},
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
|
|
170
183
|
async function queryStatus(
|
|
171
184
|
handle: ReturnType<Client["workflow"]["getHandle"]>,
|
|
172
185
|
): Promise<RunStatus | object> {
|
package/src/cli.ts
CHANGED
|
@@ -10,12 +10,20 @@ import type { ReadableStream as NodeReadableStream } from "node:stream/web";
|
|
|
10
10
|
import { fileURLToPath } from "node:url";
|
|
11
11
|
import { parseArgs } from "node:util";
|
|
12
12
|
import { buildCommit } from "./build-metadata.js";
|
|
13
|
+
import { loadWorkerConfig } from "./config.js";
|
|
13
14
|
import { runCommand } from "./process.js";
|
|
14
15
|
import { collectGitHubPullRequestProvenance, collectRepositoryProvenance } from "./provenance.js";
|
|
16
|
+
import { isExecutionBackend, isHarborEnvironment, matchingHarborEnvironment } from "./providers.js";
|
|
15
17
|
import { type PolledRunStatus, waitForRun } from "./run-wait.js";
|
|
18
|
+
import { SetupCanceledError } from "./terminal-prompts.js";
|
|
19
|
+
import { applyVercelProfile } from "./vercel-profile.js";
|
|
20
|
+
import { setupVercel } from "./vercel-setup.js";
|
|
16
21
|
|
|
17
22
|
const [command, ...rest] = process.argv.slice(2);
|
|
18
23
|
switch (command) {
|
|
24
|
+
case "setup":
|
|
25
|
+
await setup(rest);
|
|
26
|
+
break;
|
|
19
27
|
case "up":
|
|
20
28
|
await up(rest);
|
|
21
29
|
break;
|
|
@@ -47,31 +55,74 @@ switch (command) {
|
|
|
47
55
|
throw new Error(`unknown command: ${command}`);
|
|
48
56
|
}
|
|
49
57
|
|
|
58
|
+
async function setup(args: string[]): Promise<void> {
|
|
59
|
+
const [provider, ...providerArgs] = args;
|
|
60
|
+
if (provider !== "vercel") {
|
|
61
|
+
fail("setup currently supports only: self-bench setup vercel");
|
|
62
|
+
}
|
|
63
|
+
const parsed = parseArgs({
|
|
64
|
+
args: providerArgs,
|
|
65
|
+
options: {
|
|
66
|
+
profile: { type: "string", default: "default" },
|
|
67
|
+
verbose: { type: "boolean", default: false },
|
|
68
|
+
},
|
|
69
|
+
strict: true,
|
|
70
|
+
});
|
|
71
|
+
try {
|
|
72
|
+
await setupVercel({
|
|
73
|
+
profileName: parsed.values.profile,
|
|
74
|
+
verbose: parsed.values.verbose,
|
|
75
|
+
});
|
|
76
|
+
} catch (error) {
|
|
77
|
+
if (error instanceof SetupCanceledError) {
|
|
78
|
+
process.exitCode = 130;
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
throw error;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
50
85
|
async function up(args: string[]): Promise<void> {
|
|
51
86
|
const parsed = parseArgs({
|
|
52
87
|
args,
|
|
53
88
|
options: {
|
|
54
89
|
backend: { type: "string", default: "docker" },
|
|
90
|
+
"harbor-environment": { type: "string" },
|
|
55
91
|
"modal-config": { type: "string" },
|
|
92
|
+
"vercel-profile": { type: "string" },
|
|
56
93
|
},
|
|
57
94
|
strict: true,
|
|
58
95
|
});
|
|
59
|
-
if (parsed.values.backend
|
|
60
|
-
fail('--backend must be "docker" or "
|
|
96
|
+
if (!isExecutionBackend(parsed.values.backend)) {
|
|
97
|
+
fail('--backend must be "docker", "modal", or "vercel"');
|
|
61
98
|
}
|
|
62
|
-
if (
|
|
63
|
-
|
|
99
|
+
if (
|
|
100
|
+
parsed.values["harbor-environment"] !== undefined &&
|
|
101
|
+
!isHarborEnvironment(parsed.values["harbor-environment"])
|
|
102
|
+
) {
|
|
103
|
+
fail('--harbor-environment must be "docker" or "modal"');
|
|
64
104
|
}
|
|
65
105
|
|
|
66
106
|
const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
67
107
|
const composeFile = resolve(projectRoot, "compose.yaml");
|
|
68
108
|
const backend = parsed.values.backend;
|
|
69
|
-
const
|
|
109
|
+
const harborEnvironment =
|
|
110
|
+
parsed.values["harbor-environment"] ??
|
|
111
|
+
matchingHarborEnvironment(backend) ??
|
|
112
|
+
fail(`--harbor-environment is required with --backend ${backend}`);
|
|
113
|
+
const usesModal = backend === "modal" || harborEnvironment === "modal";
|
|
114
|
+
if (!usesModal && parsed.values["modal-config"] !== undefined) {
|
|
115
|
+
fail("--modal-config requires Modal generation or Harbor");
|
|
116
|
+
}
|
|
117
|
+
if (backend !== "vercel" && parsed.values["vercel-profile"] !== undefined) {
|
|
118
|
+
fail("--vercel-profile requires Vercel generation");
|
|
119
|
+
}
|
|
120
|
+
let environment: NodeJS.ProcessEnv = {
|
|
70
121
|
...process.env,
|
|
71
122
|
SELFBENCH_BUILD_COMMIT: await resolveSelfBenchCommit(),
|
|
72
123
|
SELFBENCH_EXECUTION_BACKEND: backend,
|
|
73
|
-
SELFBENCH_HARBOR_ENVIRONMENT:
|
|
74
|
-
...(
|
|
124
|
+
SELFBENCH_HARBOR_ENVIRONMENT: harborEnvironment,
|
|
125
|
+
...(usesModal
|
|
75
126
|
? {
|
|
76
127
|
SELFBENCH_MODAL_CONFIG_PATH: resolve(
|
|
77
128
|
parsed.values["modal-config"] ?? resolve(homedir(), ".modal.toml"),
|
|
@@ -79,6 +130,10 @@ async function up(args: string[]): Promise<void> {
|
|
|
79
130
|
}
|
|
80
131
|
: {}),
|
|
81
132
|
};
|
|
133
|
+
if (backend === "vercel") {
|
|
134
|
+
environment = await applyVercelProfile(environment, parsed.values["vercel-profile"]);
|
|
135
|
+
loadWorkerConfig(environment);
|
|
136
|
+
}
|
|
82
137
|
|
|
83
138
|
if (backend === "docker") {
|
|
84
139
|
await runCommand(
|
|
@@ -97,7 +152,9 @@ async function up(args: string[]): Promise<void> {
|
|
|
97
152
|
await runCommand("docker", ["compose", "--file", composeFile, "up", "-d", "--build"], {
|
|
98
153
|
env: environment,
|
|
99
154
|
});
|
|
100
|
-
console.log(
|
|
155
|
+
console.log(
|
|
156
|
+
`SelfBench is running with ${backend} generation and ${harborEnvironment} Harbor at http://127.0.0.1:8080`,
|
|
157
|
+
);
|
|
101
158
|
}
|
|
102
159
|
|
|
103
160
|
async function down(): Promise<void> {
|
|
@@ -340,7 +397,9 @@ function printHelp(): void {
|
|
|
340
397
|
console.log(`SelfBench creates durable tiered Harbor evaluations.
|
|
341
398
|
|
|
342
399
|
Usage:
|
|
343
|
-
self-bench
|
|
400
|
+
self-bench setup vercel [--profile NAME] [--verbose]
|
|
401
|
+
self-bench up [--backend docker|modal|vercel] [--harbor-environment docker|modal]
|
|
402
|
+
[--modal-config PATH] [--vercel-profile NAME]
|
|
344
403
|
self-bench down
|
|
345
404
|
self-bench run --repo PATH [--easy-count N] [--medium-count N] [--hard-count N]
|
|
346
405
|
[--model MODEL] [--run-id ID] [--wait] [--output OUTPUT.tar.gz]
|
|
@@ -349,8 +408,10 @@ Usage:
|
|
|
349
408
|
self-bench download RUN_ID OUTPUT.tar.gz
|
|
350
409
|
self-bench list
|
|
351
410
|
|
|
352
|
-
The up command starts the local stack and
|
|
353
|
-
|
|
411
|
+
The up command starts the local stack. Docker and Modal default Harbor to the matching backend; Vercel
|
|
412
|
+
requires --harbor-environment because Harbor does not support Vercel. Modal generation or Harbor uses
|
|
413
|
+
~/.modal.toml unless --modal-config overrides it. Run self-bench setup vercel once to create or select a
|
|
414
|
+
project, publish the pinned runtime image, verify access, and save an owner-only local profile.
|
|
354
415
|
|
|
355
416
|
The tier counts are candidate authoring budgets, not accepted-task targets. Rejected candidates are not
|
|
356
417
|
replaced, and the export contains only accepted tasks. The run command performs only repository metadata
|
package/src/config.ts
CHANGED
|
@@ -1,4 +1,15 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import {
|
|
3
|
+
EXECUTION_BACKENDS,
|
|
4
|
+
type ExecutionBackend,
|
|
5
|
+
HARBOR_ENVIRONMENTS,
|
|
6
|
+
type HarborEnvironment,
|
|
7
|
+
matchingHarborEnvironment,
|
|
8
|
+
} from "./providers.js";
|
|
9
|
+
import { parseSandboxTimeoutCapText, STANDARD_VERCEL_TIMEOUT_CAP_MS } from "./sandbox-timeout.js";
|
|
10
|
+
|
|
11
|
+
const emptyStringAsUndefined = (value: unknown): unknown =>
|
|
12
|
+
typeof value === "string" && value.trim() === "" ? undefined : value;
|
|
2
13
|
|
|
3
14
|
const environmentSchema = z.object({
|
|
4
15
|
SELFBENCH_API_HOST: z.string().default("127.0.0.1"),
|
|
@@ -8,12 +19,19 @@ const environmentSchema = z.object({
|
|
|
8
19
|
SELFBENCH_ARTIFACT_DIR: z.string().default(".selfbench/artifacts"),
|
|
9
20
|
SELFBENCH_GCS_BUCKET: z.string().optional(),
|
|
10
21
|
SELFBENCH_GCS_PREFIX: z.string().default("selfbench"),
|
|
11
|
-
SELFBENCH_EXECUTION_BACKEND: z.enum(
|
|
22
|
+
SELFBENCH_EXECUTION_BACKEND: z.enum(EXECUTION_BACKENDS).default("docker"),
|
|
12
23
|
SELFBENCH_DOCKER_IMAGE: z.string().default("selfbench-sandbox:local"),
|
|
13
|
-
SELFBENCH_HARBOR_ENVIRONMENT: z.
|
|
24
|
+
SELFBENCH_HARBOR_ENVIRONMENT: z.preprocess(
|
|
25
|
+
emptyStringAsUndefined,
|
|
26
|
+
z.enum(HARBOR_ENVIRONMENTS).optional(),
|
|
27
|
+
),
|
|
14
28
|
SELFBENCH_MODAL_APP: z.string().default("selfbench"),
|
|
15
29
|
SELFBENCH_MODAL_ENVIRONMENT: z.string().optional(),
|
|
16
30
|
SELFBENCH_MODAL_IMAGE: z.string().default("node:22-bookworm"),
|
|
31
|
+
SELFBENCH_VERCEL_IMAGE: z.preprocess(emptyStringAsUndefined, z.string().trim().min(1).optional()),
|
|
32
|
+
// Keep provider-specific validation in the Vercel branch so a stale Vercel
|
|
33
|
+
// variable cannot break an otherwise unrelated Docker or Modal worker.
|
|
34
|
+
SELFBENCH_VERCEL_TIMEOUT_CAP: z.preprocess(emptyStringAsUndefined, z.string().optional()),
|
|
17
35
|
SELFBENCH_BUILD_COMMIT: z.preprocess(
|
|
18
36
|
(value) => (value === "" ? undefined : value),
|
|
19
37
|
z
|
|
@@ -21,7 +39,10 @@ const environmentSchema = z.object({
|
|
|
21
39
|
.regex(/^[0-9a-f]{40}$/i)
|
|
22
40
|
.optional(),
|
|
23
41
|
),
|
|
24
|
-
SELFBENCH_ACTIVITY_CONCURRENCY: z.
|
|
42
|
+
SELFBENCH_ACTIVITY_CONCURRENCY: z.preprocess(
|
|
43
|
+
emptyStringAsUndefined,
|
|
44
|
+
z.coerce.number().int().min(1).max(100).optional(),
|
|
45
|
+
),
|
|
25
46
|
SELFBENCH_TASK_QUEUE: z.string().default("selfbench-dev"),
|
|
26
47
|
SELFBENCH_TEMPORAL_ADDRESS: z.string().default("127.0.0.1:7233"),
|
|
27
48
|
SELFBENCH_TEMPORAL_NAMESPACE: z.string().default("default"),
|
|
@@ -32,6 +53,37 @@ const environmentSchema = z.object({
|
|
|
32
53
|
.default(false),
|
|
33
54
|
});
|
|
34
55
|
|
|
56
|
+
export interface VercelCredentials {
|
|
57
|
+
readonly token: string;
|
|
58
|
+
readonly teamId: string;
|
|
59
|
+
readonly projectId: string;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
type ExecutionConfig =
|
|
63
|
+
| { readonly kind: "docker"; readonly image: string }
|
|
64
|
+
| {
|
|
65
|
+
readonly kind: "modal";
|
|
66
|
+
readonly app: string;
|
|
67
|
+
readonly environment?: string;
|
|
68
|
+
readonly image: string;
|
|
69
|
+
}
|
|
70
|
+
| { readonly kind: "vercel"; readonly image: string; readonly timeoutCapMs: number };
|
|
71
|
+
|
|
72
|
+
type WorkerExecutionConfig =
|
|
73
|
+
| Exclude<ExecutionConfig, { readonly kind: "vercel" }>
|
|
74
|
+
| {
|
|
75
|
+
readonly kind: "vercel";
|
|
76
|
+
readonly image: string;
|
|
77
|
+
readonly timeoutCapMs: number;
|
|
78
|
+
readonly credentials: VercelCredentials;
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
const DEFAULT_ACTIVITY_CONCURRENCY = {
|
|
82
|
+
docker: 1,
|
|
83
|
+
modal: 20,
|
|
84
|
+
vercel: 4,
|
|
85
|
+
} as const satisfies Record<ExecutionBackend, number>;
|
|
86
|
+
|
|
35
87
|
export interface SelfBenchConfig {
|
|
36
88
|
readonly apiHost: string;
|
|
37
89
|
readonly apiPort: number;
|
|
@@ -41,15 +93,8 @@ export interface SelfBenchConfig {
|
|
|
41
93
|
readonly artifact:
|
|
42
94
|
| { readonly kind: "local"; readonly directory: string }
|
|
43
95
|
| { readonly kind: "gcs"; readonly bucket: string; readonly prefix: string };
|
|
44
|
-
readonly execution:
|
|
45
|
-
|
|
46
|
-
| {
|
|
47
|
-
readonly kind: "modal";
|
|
48
|
-
readonly app: string;
|
|
49
|
-
readonly environment?: string;
|
|
50
|
-
readonly image: string;
|
|
51
|
-
};
|
|
52
|
-
readonly harborEnvironment: "docker" | "modal";
|
|
96
|
+
readonly execution: ExecutionConfig;
|
|
97
|
+
readonly harborEnvironment: HarborEnvironment;
|
|
53
98
|
readonly temporal: {
|
|
54
99
|
readonly address: string;
|
|
55
100
|
readonly namespace: string;
|
|
@@ -59,6 +104,14 @@ export interface SelfBenchConfig {
|
|
|
59
104
|
};
|
|
60
105
|
}
|
|
61
106
|
|
|
107
|
+
export interface SelfBenchWorkerConfig extends Omit<SelfBenchConfig, "execution"> {
|
|
108
|
+
readonly execution: WorkerExecutionConfig;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function isDigestPinnedOciImage(image: string): boolean {
|
|
112
|
+
return /^[^@\s]+@sha256:[0-9a-f]{64}$/i.test(image);
|
|
113
|
+
}
|
|
114
|
+
|
|
62
115
|
export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBenchConfig {
|
|
63
116
|
const value = environmentSchema.parse(environment);
|
|
64
117
|
if (
|
|
@@ -76,6 +129,43 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBe
|
|
|
76
129
|
}
|
|
77
130
|
: { kind: "local" as const, directory: value.SELFBENCH_ARTIFACT_DIR };
|
|
78
131
|
|
|
132
|
+
const harborEnvironment =
|
|
133
|
+
value.SELFBENCH_HARBOR_ENVIRONMENT ??
|
|
134
|
+
matchingHarborEnvironment(value.SELFBENCH_EXECUTION_BACKEND) ??
|
|
135
|
+
fail(
|
|
136
|
+
`SELFBENCH_HARBOR_ENVIRONMENT is required for ${value.SELFBENCH_EXECUTION_BACKEND} execution`,
|
|
137
|
+
);
|
|
138
|
+
let execution: ExecutionConfig;
|
|
139
|
+
switch (value.SELFBENCH_EXECUTION_BACKEND) {
|
|
140
|
+
case "docker":
|
|
141
|
+
execution = { kind: "docker", image: value.SELFBENCH_DOCKER_IMAGE };
|
|
142
|
+
break;
|
|
143
|
+
case "modal":
|
|
144
|
+
execution = {
|
|
145
|
+
kind: "modal",
|
|
146
|
+
app: value.SELFBENCH_MODAL_APP,
|
|
147
|
+
image: value.SELFBENCH_MODAL_IMAGE,
|
|
148
|
+
...(value.SELFBENCH_MODAL_ENVIRONMENT
|
|
149
|
+
? { environment: value.SELFBENCH_MODAL_ENVIRONMENT }
|
|
150
|
+
: {}),
|
|
151
|
+
};
|
|
152
|
+
break;
|
|
153
|
+
case "vercel": {
|
|
154
|
+
const image =
|
|
155
|
+
value.SELFBENCH_VERCEL_IMAGE ??
|
|
156
|
+
fail("SELFBENCH_VERCEL_IMAGE is required for Vercel execution");
|
|
157
|
+
if (!isDigestPinnedOciImage(image)) {
|
|
158
|
+
fail("SELFBENCH_VERCEL_IMAGE must be pinned by sha256 digest");
|
|
159
|
+
}
|
|
160
|
+
execution = {
|
|
161
|
+
kind: "vercel",
|
|
162
|
+
image,
|
|
163
|
+
timeoutCapMs: vercelTimeoutCap(value.SELFBENCH_VERCEL_TIMEOUT_CAP),
|
|
164
|
+
};
|
|
165
|
+
break;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
79
169
|
return {
|
|
80
170
|
apiHost: value.SELFBENCH_API_HOST,
|
|
81
171
|
apiPort: value.SELFBENCH_API_PORT,
|
|
@@ -85,20 +175,10 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBe
|
|
|
85
175
|
: {}),
|
|
86
176
|
activityConcurrency:
|
|
87
177
|
value.SELFBENCH_ACTIVITY_CONCURRENCY ??
|
|
88
|
-
|
|
178
|
+
DEFAULT_ACTIVITY_CONCURRENCY[value.SELFBENCH_EXECUTION_BACKEND],
|
|
89
179
|
artifact,
|
|
90
|
-
harborEnvironment
|
|
91
|
-
execution
|
|
92
|
-
value.SELFBENCH_EXECUTION_BACKEND === "modal"
|
|
93
|
-
? {
|
|
94
|
-
kind: "modal",
|
|
95
|
-
app: value.SELFBENCH_MODAL_APP,
|
|
96
|
-
image: value.SELFBENCH_MODAL_IMAGE,
|
|
97
|
-
...(value.SELFBENCH_MODAL_ENVIRONMENT
|
|
98
|
-
? { environment: value.SELFBENCH_MODAL_ENVIRONMENT }
|
|
99
|
-
: {}),
|
|
100
|
-
}
|
|
101
|
-
: { kind: "docker", image: value.SELFBENCH_DOCKER_IMAGE },
|
|
180
|
+
harborEnvironment,
|
|
181
|
+
execution,
|
|
102
182
|
temporal: {
|
|
103
183
|
address: value.SELFBENCH_TEMPORAL_ADDRESS,
|
|
104
184
|
namespace: value.SELFBENCH_TEMPORAL_NAMESPACE,
|
|
@@ -109,6 +189,51 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBe
|
|
|
109
189
|
};
|
|
110
190
|
}
|
|
111
191
|
|
|
192
|
+
export function loadWorkerConfig(
|
|
193
|
+
environment: NodeJS.ProcessEnv = process.env,
|
|
194
|
+
): SelfBenchWorkerConfig {
|
|
195
|
+
const config = loadConfig(environment);
|
|
196
|
+
if (config.execution.kind !== "vercel") {
|
|
197
|
+
return { ...config, execution: config.execution };
|
|
198
|
+
}
|
|
199
|
+
return {
|
|
200
|
+
...config,
|
|
201
|
+
execution: {
|
|
202
|
+
...config.execution,
|
|
203
|
+
credentials: vercelCredentials(environment),
|
|
204
|
+
},
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
export function defaultStandaloneConcurrency(
|
|
209
|
+
config: Pick<SelfBenchConfig, "activityConcurrency" | "execution">,
|
|
210
|
+
existingDefault: number,
|
|
211
|
+
): number {
|
|
212
|
+
return config.execution.kind === "vercel" ? config.activityConcurrency : existingDefault;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function vercelCredentials(environment: NodeJS.ProcessEnv): VercelCredentials {
|
|
216
|
+
const token = environment.VERCEL_TOKEN?.trim();
|
|
217
|
+
const teamId = environment.VERCEL_TEAM_ID?.trim();
|
|
218
|
+
const projectId = environment.VERCEL_PROJECT_ID?.trim();
|
|
219
|
+
if (!token || !teamId || !projectId) {
|
|
220
|
+
fail("VERCEL_TOKEN, VERCEL_TEAM_ID, and VERCEL_PROJECT_ID are required for Vercel execution");
|
|
221
|
+
}
|
|
222
|
+
return { token, teamId, projectId };
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function vercelTimeoutCap(value: string | undefined): number {
|
|
226
|
+
if (value === undefined) {
|
|
227
|
+
return STANDARD_VERCEL_TIMEOUT_CAP_MS;
|
|
228
|
+
}
|
|
229
|
+
return z
|
|
230
|
+
.number()
|
|
231
|
+
.int()
|
|
232
|
+
.min(100)
|
|
233
|
+
.max(STANDARD_VERCEL_TIMEOUT_CAP_MS)
|
|
234
|
+
.parse(parseSandboxTimeoutCapText(value));
|
|
235
|
+
}
|
|
236
|
+
|
|
112
237
|
function fail(message: string): never {
|
|
113
238
|
throw new Error(message);
|
|
114
239
|
}
|
package/src/contracts.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { EXECUTION_BACKENDS, HARBOR_ENVIRONMENTS, matchingHarborEnvironment } from "./providers.js";
|
|
2
3
|
|
|
3
4
|
export const commitSchema = z.string().regex(/^[0-9a-f]{40}$/i, "expected a full commit SHA");
|
|
4
5
|
|
|
@@ -34,6 +35,37 @@ const candidateCountsSchema = z
|
|
|
34
35
|
message: "at most 100 candidates may be requested",
|
|
35
36
|
});
|
|
36
37
|
|
|
38
|
+
const runVersionSchema = z
|
|
39
|
+
.object({
|
|
40
|
+
selfbenchCommit: commitSchema,
|
|
41
|
+
executionBackend: z.enum(EXECUTION_BACKENDS),
|
|
42
|
+
harborEnvironment: z.enum(HARBOR_ENVIRONMENTS).optional(),
|
|
43
|
+
sandboxImage: z.string().min(1),
|
|
44
|
+
sandboxTimeoutCapMs: z.number().int().min(100).optional(),
|
|
45
|
+
schema: z.literal(1),
|
|
46
|
+
})
|
|
47
|
+
.transform((version, context) => {
|
|
48
|
+
const harborEnvironment =
|
|
49
|
+
version.harborEnvironment ?? matchingHarborEnvironment(version.executionBackend);
|
|
50
|
+
if (!harborEnvironment) {
|
|
51
|
+
context.addIssue({
|
|
52
|
+
code: "custom",
|
|
53
|
+
message: `harborEnvironment is required for ${version.executionBackend} execution`,
|
|
54
|
+
path: ["harborEnvironment"],
|
|
55
|
+
});
|
|
56
|
+
return z.NEVER;
|
|
57
|
+
}
|
|
58
|
+
if (version.sandboxTimeoutCapMs !== undefined && version.executionBackend !== "vercel") {
|
|
59
|
+
context.addIssue({
|
|
60
|
+
code: "custom",
|
|
61
|
+
message: "sandboxTimeoutCapMs is only valid for Vercel execution",
|
|
62
|
+
path: ["sandboxTimeoutCapMs"],
|
|
63
|
+
});
|
|
64
|
+
return z.NEVER;
|
|
65
|
+
}
|
|
66
|
+
return { ...version, harborEnvironment };
|
|
67
|
+
});
|
|
68
|
+
|
|
37
69
|
export const runRequestSchema = z.object({
|
|
38
70
|
runId: z.string().regex(/^[a-z0-9][a-z0-9-]{2,62}$/),
|
|
39
71
|
repository: repositoryRefSchema,
|
|
@@ -44,12 +76,7 @@ export const runRequestSchema = z.object({
|
|
|
44
76
|
model: z.string().min(1),
|
|
45
77
|
reasoningEffort: z.literal("high"),
|
|
46
78
|
}),
|
|
47
|
-
version:
|
|
48
|
-
selfbenchCommit: commitSchema,
|
|
49
|
-
executionBackend: z.enum(["docker", "modal"]),
|
|
50
|
-
sandboxImage: z.string().min(1),
|
|
51
|
-
schema: z.literal(1),
|
|
52
|
-
}),
|
|
79
|
+
version: runVersionSchema,
|
|
53
80
|
});
|
|
54
81
|
|
|
55
82
|
export type RunRequest = z.infer<typeof runRequestSchema>;
|
package/src/eval-main.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import { parseArgs } from "node:util";
|
|
4
4
|
import { MATRIX_MODELS, type MatrixModel, runMatrix } from "./evaluate.js";
|
|
5
|
+
import { isHarborEnvironment } from "./providers.js";
|
|
5
6
|
|
|
6
7
|
const parsed = parseArgs({
|
|
7
8
|
options: {
|
|
@@ -33,8 +34,8 @@ Options:
|
|
|
33
34
|
-h, --help Show this help`);
|
|
34
35
|
process.exit(0);
|
|
35
36
|
}
|
|
36
|
-
const environment = parsed.values.environment;
|
|
37
|
-
if (environment
|
|
37
|
+
const environment = parsed.values.environment ?? "modal";
|
|
38
|
+
if (!isHarborEnvironment(environment)) {
|
|
38
39
|
throw new Error("--environment must be docker or modal");
|
|
39
40
|
}
|
|
40
41
|
const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
|
package/src/evaluate.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
2
2
|
import { homedir, tmpdir } from "node:os";
|
|
3
3
|
import { basename, join, resolve } from "node:path";
|
|
4
|
+
import { harborChildEnvironment } from "./harbor-environment.js";
|
|
4
5
|
import { archiveIncompleteHarborJob, tryReadHarborJobResult } from "./harbor-results.js";
|
|
5
6
|
import { parallelMap } from "./parallel.js";
|
|
6
7
|
import { runCommand } from "./process.js";
|
|
8
|
+
import type { HarborEnvironment } from "./providers.js";
|
|
7
9
|
import { assertCodexSubscriptionAuth, openAiApiKey } from "./subscription-auth.js";
|
|
8
10
|
|
|
9
11
|
export const MATRIX_MODELS = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"] as const;
|
|
@@ -15,7 +17,7 @@ export interface MatrixOptions {
|
|
|
15
17
|
readonly tasksPath?: string;
|
|
16
18
|
readonly jobsDirectory: string;
|
|
17
19
|
readonly harborPath?: string;
|
|
18
|
-
readonly environment?:
|
|
20
|
+
readonly environment?: HarborEnvironment;
|
|
19
21
|
readonly concurrency?: number;
|
|
20
22
|
readonly authPath?: string;
|
|
21
23
|
readonly models?: readonly MatrixModel[];
|
|
@@ -114,7 +116,7 @@ async function runTrial(input: {
|
|
|
114
116
|
readonly model: MatrixModel;
|
|
115
117
|
readonly jobsDirectory: string;
|
|
116
118
|
readonly harborPath: string;
|
|
117
|
-
readonly environment:
|
|
119
|
+
readonly environment: HarborEnvironment;
|
|
118
120
|
readonly authPath?: string;
|
|
119
121
|
}): Promise<MatrixTrialSummary> {
|
|
120
122
|
const taskId = basename(input.taskDirectory);
|
|
@@ -125,7 +127,7 @@ async function runTrial(input: {
|
|
|
125
127
|
}
|
|
126
128
|
await archiveIncompleteHarborJob(input.jobsDirectory, jobName);
|
|
127
129
|
|
|
128
|
-
const environment =
|
|
130
|
+
const environment = harborChildEnvironment();
|
|
129
131
|
if (input.authPath) {
|
|
130
132
|
delete environment.OPENAI_API_KEY;
|
|
131
133
|
environment.CODEX_FORCE_AUTH_JSON = "1";
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
const VERCEL_CONTROL_CREDENTIALS = [
|
|
2
|
+
"VERCEL_AUTH_TOKEN",
|
|
3
|
+
"VERCEL_TOKEN",
|
|
4
|
+
"VERCEL_TEAM_ID",
|
|
5
|
+
"VERCEL_PROJECT_ID",
|
|
6
|
+
"VERCEL_OIDC_TOKEN",
|
|
7
|
+
] as const;
|
|
8
|
+
|
|
9
|
+
export function harborChildEnvironment(
|
|
10
|
+
environment: NodeJS.ProcessEnv = process.env,
|
|
11
|
+
): NodeJS.ProcessEnv {
|
|
12
|
+
const child = { ...environment };
|
|
13
|
+
for (const key of VERCEL_CONTROL_CREDENTIALS) {
|
|
14
|
+
delete child[key];
|
|
15
|
+
}
|
|
16
|
+
return child;
|
|
17
|
+
}
|
package/src/harbor-results.ts
CHANGED
|
@@ -20,6 +20,8 @@ const infrastructurePatterns = [
|
|
|
20
20
|
/modal.*(?:unavailable|timed out|timeout)/i,
|
|
21
21
|
/image build for im-[a-z0-9]+ failed/i,
|
|
22
22
|
/all predefined address pools have been fully subnetted/i,
|
|
23
|
+
/mounts denied/i,
|
|
24
|
+
/network-policy[\s\S]*could not process rule/i,
|
|
23
25
|
];
|
|
24
26
|
|
|
25
27
|
class IncompleteHarborJobError extends Error {}
|
package/src/process.ts
CHANGED
|
@@ -6,6 +6,8 @@ export interface CommandResult {
|
|
|
6
6
|
readonly exitCode: number;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
+
export type CommandOutputHandler = (stream: "stdout" | "stderr", chunk: Uint8Array) => void;
|
|
10
|
+
|
|
9
11
|
export interface CommandOptions {
|
|
10
12
|
readonly cwd?: string;
|
|
11
13
|
readonly env?: NodeJS.ProcessEnv;
|
|
@@ -14,7 +16,7 @@ export interface CommandOptions {
|
|
|
14
16
|
readonly inactivityTimeoutMs?: number;
|
|
15
17
|
readonly allowFailure?: boolean;
|
|
16
18
|
readonly signal?: AbortSignal;
|
|
17
|
-
readonly onOutput?:
|
|
19
|
+
readonly onOutput?: CommandOutputHandler;
|
|
18
20
|
}
|
|
19
21
|
|
|
20
22
|
export class InactivityTimeoutError extends Error {
|
package/src/providers.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export const EXECUTION_BACKENDS = ["docker", "modal", "vercel"] as const;
|
|
2
|
+
export type ExecutionBackend = (typeof EXECUTION_BACKENDS)[number];
|
|
3
|
+
|
|
4
|
+
export const HARBOR_ENVIRONMENTS = ["docker", "modal"] as const;
|
|
5
|
+
export type HarborEnvironment = (typeof HARBOR_ENVIRONMENTS)[number];
|
|
6
|
+
|
|
7
|
+
const executionBackends: ReadonlySet<string> = new Set(EXECUTION_BACKENDS);
|
|
8
|
+
const harborEnvironments: ReadonlySet<string> = new Set(HARBOR_ENVIRONMENTS);
|
|
9
|
+
|
|
10
|
+
export function isExecutionBackend(value: string): value is ExecutionBackend {
|
|
11
|
+
return executionBackends.has(value);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function isHarborEnvironment(value: string): value is HarborEnvironment {
|
|
15
|
+
return harborEnvironments.has(value);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function matchingHarborEnvironment(
|
|
19
|
+
backend: ExecutionBackend,
|
|
20
|
+
): HarborEnvironment | undefined {
|
|
21
|
+
return isHarborEnvironment(backend) ? backend : undefined;
|
|
22
|
+
}
|
package/src/reaudit-main.ts
CHANGED
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
couplingReviewInput,
|
|
11
11
|
couplingReviewSchema,
|
|
12
12
|
} from "./codex-review.js";
|
|
13
|
-
import {
|
|
13
|
+
import { defaultStandaloneConcurrency, loadWorkerConfig } from "./config.js";
|
|
14
14
|
import {
|
|
15
15
|
buildCouplingEvidence,
|
|
16
16
|
discoverContractArtifacts,
|
|
@@ -27,7 +27,7 @@ const parsed = parseArgs({
|
|
|
27
27
|
tasks: { type: "string" },
|
|
28
28
|
task: { type: "string" },
|
|
29
29
|
output: { type: "string" },
|
|
30
|
-
concurrency: { type: "string"
|
|
30
|
+
concurrency: { type: "string" },
|
|
31
31
|
help: { type: "boolean", short: "h" },
|
|
32
32
|
},
|
|
33
33
|
strict: true,
|
|
@@ -39,7 +39,7 @@ Usage:
|
|
|
39
39
|
self-bench-reaudit --tasks DIRECTORY --output REPORT.json [options]
|
|
40
40
|
|
|
41
41
|
Options:
|
|
42
|
-
--concurrency N Concurrent Sol reviews (default: 10)
|
|
42
|
+
--concurrency N Concurrent Sol reviews (default: 10; Vercel: worker setting, default 4)
|
|
43
43
|
--task ID Review only one task ID
|
|
44
44
|
-h, --help Show this help`);
|
|
45
45
|
process.exit(0);
|
|
@@ -47,8 +47,11 @@ Options:
|
|
|
47
47
|
|
|
48
48
|
const tasksDirectory = resolve(parsed.values.tasks ?? fail("--tasks is required"));
|
|
49
49
|
const outputPath = resolve(parsed.values.output ?? fail("--output is required"));
|
|
50
|
-
const
|
|
51
|
-
const
|
|
50
|
+
const config = loadWorkerConfig();
|
|
51
|
+
const concurrency = positiveInteger(
|
|
52
|
+
parsed.values.concurrency ?? String(defaultStandaloneConcurrency(config, 10)),
|
|
53
|
+
"--concurrency",
|
|
54
|
+
);
|
|
52
55
|
const sandbox = createSandboxExecutor(config.execution);
|
|
53
56
|
const [taskIds, modelAuth, reviewer] = await Promise.all([
|
|
54
57
|
readdir(tasksDirectory),
|