self-bench 0.3.3 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile.sandbox +1 -1
- package/README.md +20 -0
- package/bun.lock +48 -0
- package/compose.yaml +7 -1
- package/dist/agent-smoke-main.js +3 -2
- package/dist/agent-smoke-main.js.map +1 -1
- package/dist/agent-smoke.d.ts +2 -1
- package/dist/agent-smoke.d.ts.map +1 -1
- package/dist/agent-smoke.js +6 -1
- package/dist/agent-smoke.js.map +1 -1
- package/dist/api.d.ts +24 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +25 -18
- package/dist/api.js.map +1 -1
- package/dist/build-metadata.js +1 -1
- package/dist/cli.js +67 -11
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +36 -10
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +89 -15
- package/dist/config.js.map +1 -1
- package/dist/contracts.d.ts +22 -2
- package/dist/contracts.d.ts.map +1 -1
- package/dist/contracts.js +31 -6
- package/dist/contracts.js.map +1 -1
- package/dist/eval-main.js +3 -2
- package/dist/eval-main.js.map +1 -1
- package/dist/evaluate.d.ts +2 -1
- package/dist/evaluate.d.ts.map +1 -1
- package/dist/evaluate.js +2 -1
- package/dist/evaluate.js.map +1 -1
- package/dist/harbor-environment.d.ts +2 -0
- package/dist/harbor-environment.d.ts.map +1 -0
- package/dist/harbor-environment.js +15 -0
- package/dist/harbor-environment.js.map +1 -0
- package/dist/harbor-results.d.ts.map +1 -1
- package/dist/harbor-results.js +2 -0
- package/dist/harbor-results.js.map +1 -1
- package/dist/process.d.ts +2 -1
- package/dist/process.d.ts.map +1 -1
- package/dist/process.js.map +1 -1
- package/dist/providers.d.ts +8 -0
- package/dist/providers.d.ts.map +1 -0
- package/dist/providers.js +14 -0
- package/dist/providers.js.map +1 -0
- package/dist/reaudit-main.js +5 -5
- package/dist/reaudit-main.js.map +1 -1
- package/dist/repair-main.js +5 -5
- package/dist/repair-main.js.map +1 -1
- package/dist/sandbox-author.bundle.js +40 -6
- package/dist/sandbox-timeout.d.ts +11 -0
- package/dist/sandbox-timeout.d.ts.map +1 -0
- package/dist/sandbox-timeout.js +41 -0
- package/dist/sandbox-timeout.js.map +1 -0
- package/dist/sandbox-validation-repair.bundle.js +40 -6
- package/dist/sandbox.d.ts +2 -2
- package/dist/sandbox.d.ts.map +1 -1
- package/dist/sandbox.js +10 -3
- package/dist/sandbox.js.map +1 -1
- package/dist/temporal/activities.d.ts +2 -2
- package/dist/temporal/activities.d.ts.map +1 -1
- package/dist/temporal/activities.js +7 -1
- package/dist/temporal/activities.js.map +1 -1
- package/dist/temporal/worker-main.js +2 -2
- package/dist/temporal/worker-main.js.map +1 -1
- package/dist/terminal-prompts.d.ts +26 -0
- package/dist/terminal-prompts.d.ts.map +1 -0
- package/dist/terminal-prompts.js +137 -0
- package/dist/terminal-prompts.js.map +1 -0
- package/dist/terminal-reporter.d.ts +30 -0
- package/dist/terminal-reporter.d.ts.map +1 -0
- package/dist/terminal-reporter.js +90 -0
- package/dist/terminal-reporter.js.map +1 -0
- package/dist/validate-main.js +8 -2
- package/dist/validate-main.js.map +1 -1
- package/dist/vercel-cli.d.ts +64 -0
- package/dist/vercel-cli.d.ts.map +1 -0
- package/dist/vercel-cli.js +303 -0
- package/dist/vercel-cli.js.map +1 -0
- package/dist/vercel-command.d.ts +14 -0
- package/dist/vercel-command.d.ts.map +1 -0
- package/dist/vercel-command.js +93 -0
- package/dist/vercel-command.js.map +1 -0
- package/dist/vercel-executor.d.ts +14 -0
- package/dist/vercel-executor.d.ts.map +1 -0
- package/dist/vercel-executor.js +356 -0
- package/dist/vercel-executor.js.map +1 -0
- package/dist/vercel-fetch.d.ts +6 -0
- package/dist/vercel-fetch.d.ts.map +1 -0
- package/dist/vercel-fetch.js +39 -0
- package/dist/vercel-fetch.js.map +1 -0
- package/dist/vercel-profile.d.ts +34 -0
- package/dist/vercel-profile.d.ts.map +1 -0
- package/dist/vercel-profile.js +244 -0
- package/dist/vercel-profile.js.map +1 -0
- package/dist/vercel-runtime-image.d.ts +51 -0
- package/dist/vercel-runtime-image.d.ts.map +1 -0
- package/dist/vercel-runtime-image.js +123 -0
- package/dist/vercel-runtime-image.js.map +1 -0
- package/dist/vercel-setup-probe.d.ts +49 -0
- package/dist/vercel-setup-probe.d.ts.map +1 -0
- package/dist/vercel-setup-probe.js +151 -0
- package/dist/vercel-setup-probe.js.map +1 -0
- package/dist/vercel-setup.d.ts +42 -0
- package/dist/vercel-setup.d.ts.map +1 -0
- package/dist/vercel-setup.js +280 -0
- package/dist/vercel-setup.js.map +1 -0
- package/docs/evaluations.md +2 -0
- package/docs/operations.md +128 -6
- package/package.json +5 -1
- package/review/index.html +14 -0
- package/review/src/App.tsx +338 -0
- package/review/src/export-loader.test.ts +57 -0
- package/review/src/export-loader.ts +151 -0
- package/review/src/main.tsx +15 -0
- package/review/src/styles.css +162 -0
- package/review/src/types.ts +47 -0
- package/review/tsconfig.json +16 -0
- package/review/vite.config.ts +20 -0
- package/scripts/verify-package.ts +27 -3
- package/src/agent-smoke-main.ts +3 -2
- package/src/agent-smoke.ts +8 -2
- package/src/api.ts +31 -18
- package/src/cli.ts +72 -11
- package/src/config.ts +150 -25
- package/src/contracts.ts +33 -6
- package/src/eval-main.ts +3 -2
- package/src/evaluate.ts +5 -3
- package/src/harbor-environment.ts +17 -0
- package/src/harbor-results.ts +2 -0
- package/src/process.ts +3 -1
- package/src/providers.ts +22 -0
- package/src/reaudit-main.ts +8 -5
- package/src/repair-main.ts +8 -5
- package/src/sandbox-timeout.ts +56 -0
- package/src/sandbox.ts +15 -5
- package/src/temporal/activities.ts +9 -3
- package/src/temporal/worker-main.ts +2 -2
- package/src/terminal-prompts.ts +197 -0
- package/src/terminal-reporter.ts +126 -0
- package/src/validate-main.ts +9 -3
- package/src/vercel-cli.ts +403 -0
- package/src/vercel-command.ts +127 -0
- package/src/vercel-executor.ts +448 -0
- package/src/vercel-fetch.ts +45 -0
- package/src/vercel-profile.ts +319 -0
- package/src/vercel-runtime-image.ts +204 -0
- package/src/vercel-setup-probe.ts +229 -0
- package/src/vercel-setup.ts +419 -0
package/src/config.ts
CHANGED
|
@@ -1,4 +1,15 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import {
|
|
3
|
+
EXECUTION_BACKENDS,
|
|
4
|
+
type ExecutionBackend,
|
|
5
|
+
HARBOR_ENVIRONMENTS,
|
|
6
|
+
type HarborEnvironment,
|
|
7
|
+
matchingHarborEnvironment,
|
|
8
|
+
} from "./providers.js";
|
|
9
|
+
import { parseSandboxTimeoutCapText, STANDARD_VERCEL_TIMEOUT_CAP_MS } from "./sandbox-timeout.js";
|
|
10
|
+
|
|
11
|
+
const emptyStringAsUndefined = (value: unknown): unknown =>
|
|
12
|
+
typeof value === "string" && value.trim() === "" ? undefined : value;
|
|
2
13
|
|
|
3
14
|
const environmentSchema = z.object({
|
|
4
15
|
SELFBENCH_API_HOST: z.string().default("127.0.0.1"),
|
|
@@ -8,12 +19,19 @@ const environmentSchema = z.object({
|
|
|
8
19
|
SELFBENCH_ARTIFACT_DIR: z.string().default(".selfbench/artifacts"),
|
|
9
20
|
SELFBENCH_GCS_BUCKET: z.string().optional(),
|
|
10
21
|
SELFBENCH_GCS_PREFIX: z.string().default("selfbench"),
|
|
11
|
-
SELFBENCH_EXECUTION_BACKEND: z.enum(
|
|
22
|
+
SELFBENCH_EXECUTION_BACKEND: z.enum(EXECUTION_BACKENDS).default("docker"),
|
|
12
23
|
SELFBENCH_DOCKER_IMAGE: z.string().default("selfbench-sandbox:local"),
|
|
13
|
-
SELFBENCH_HARBOR_ENVIRONMENT: z.
|
|
24
|
+
SELFBENCH_HARBOR_ENVIRONMENT: z.preprocess(
|
|
25
|
+
emptyStringAsUndefined,
|
|
26
|
+
z.enum(HARBOR_ENVIRONMENTS).optional(),
|
|
27
|
+
),
|
|
14
28
|
SELFBENCH_MODAL_APP: z.string().default("selfbench"),
|
|
15
29
|
SELFBENCH_MODAL_ENVIRONMENT: z.string().optional(),
|
|
16
30
|
SELFBENCH_MODAL_IMAGE: z.string().default("node:22-bookworm"),
|
|
31
|
+
SELFBENCH_VERCEL_IMAGE: z.preprocess(emptyStringAsUndefined, z.string().trim().min(1).optional()),
|
|
32
|
+
// Keep provider-specific validation in the Vercel branch so a stale Vercel
|
|
33
|
+
// variable cannot break an otherwise unrelated Docker or Modal worker.
|
|
34
|
+
SELFBENCH_VERCEL_TIMEOUT_CAP: z.preprocess(emptyStringAsUndefined, z.string().optional()),
|
|
17
35
|
SELFBENCH_BUILD_COMMIT: z.preprocess(
|
|
18
36
|
(value) => (value === "" ? undefined : value),
|
|
19
37
|
z
|
|
@@ -21,7 +39,10 @@ const environmentSchema = z.object({
|
|
|
21
39
|
.regex(/^[0-9a-f]{40}$/i)
|
|
22
40
|
.optional(),
|
|
23
41
|
),
|
|
24
|
-
SELFBENCH_ACTIVITY_CONCURRENCY: z.
|
|
42
|
+
SELFBENCH_ACTIVITY_CONCURRENCY: z.preprocess(
|
|
43
|
+
emptyStringAsUndefined,
|
|
44
|
+
z.coerce.number().int().min(1).max(100).optional(),
|
|
45
|
+
),
|
|
25
46
|
SELFBENCH_TASK_QUEUE: z.string().default("selfbench-dev"),
|
|
26
47
|
SELFBENCH_TEMPORAL_ADDRESS: z.string().default("127.0.0.1:7233"),
|
|
27
48
|
SELFBENCH_TEMPORAL_NAMESPACE: z.string().default("default"),
|
|
@@ -32,6 +53,37 @@ const environmentSchema = z.object({
|
|
|
32
53
|
.default(false),
|
|
33
54
|
});
|
|
34
55
|
|
|
56
|
+
export interface VercelCredentials {
|
|
57
|
+
readonly token: string;
|
|
58
|
+
readonly teamId: string;
|
|
59
|
+
readonly projectId: string;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
type ExecutionConfig =
|
|
63
|
+
| { readonly kind: "docker"; readonly image: string }
|
|
64
|
+
| {
|
|
65
|
+
readonly kind: "modal";
|
|
66
|
+
readonly app: string;
|
|
67
|
+
readonly environment?: string;
|
|
68
|
+
readonly image: string;
|
|
69
|
+
}
|
|
70
|
+
| { readonly kind: "vercel"; readonly image: string; readonly timeoutCapMs: number };
|
|
71
|
+
|
|
72
|
+
type WorkerExecutionConfig =
|
|
73
|
+
| Exclude<ExecutionConfig, { readonly kind: "vercel" }>
|
|
74
|
+
| {
|
|
75
|
+
readonly kind: "vercel";
|
|
76
|
+
readonly image: string;
|
|
77
|
+
readonly timeoutCapMs: number;
|
|
78
|
+
readonly credentials: VercelCredentials;
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
const DEFAULT_ACTIVITY_CONCURRENCY = {
|
|
82
|
+
docker: 1,
|
|
83
|
+
modal: 20,
|
|
84
|
+
vercel: 4,
|
|
85
|
+
} as const satisfies Record<ExecutionBackend, number>;
|
|
86
|
+
|
|
35
87
|
export interface SelfBenchConfig {
|
|
36
88
|
readonly apiHost: string;
|
|
37
89
|
readonly apiPort: number;
|
|
@@ -41,15 +93,8 @@ export interface SelfBenchConfig {
|
|
|
41
93
|
readonly artifact:
|
|
42
94
|
| { readonly kind: "local"; readonly directory: string }
|
|
43
95
|
| { readonly kind: "gcs"; readonly bucket: string; readonly prefix: string };
|
|
44
|
-
readonly execution:
|
|
45
|
-
|
|
46
|
-
| {
|
|
47
|
-
readonly kind: "modal";
|
|
48
|
-
readonly app: string;
|
|
49
|
-
readonly environment?: string;
|
|
50
|
-
readonly image: string;
|
|
51
|
-
};
|
|
52
|
-
readonly harborEnvironment: "docker" | "modal";
|
|
96
|
+
readonly execution: ExecutionConfig;
|
|
97
|
+
readonly harborEnvironment: HarborEnvironment;
|
|
53
98
|
readonly temporal: {
|
|
54
99
|
readonly address: string;
|
|
55
100
|
readonly namespace: string;
|
|
@@ -59,6 +104,14 @@ export interface SelfBenchConfig {
|
|
|
59
104
|
};
|
|
60
105
|
}
|
|
61
106
|
|
|
107
|
+
export interface SelfBenchWorkerConfig extends Omit<SelfBenchConfig, "execution"> {
|
|
108
|
+
readonly execution: WorkerExecutionConfig;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function isDigestPinnedOciImage(image: string): boolean {
|
|
112
|
+
return /^[^@\s]+@sha256:[0-9a-f]{64}$/i.test(image);
|
|
113
|
+
}
|
|
114
|
+
|
|
62
115
|
export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBenchConfig {
|
|
63
116
|
const value = environmentSchema.parse(environment);
|
|
64
117
|
if (
|
|
@@ -76,6 +129,43 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBe
|
|
|
76
129
|
}
|
|
77
130
|
: { kind: "local" as const, directory: value.SELFBENCH_ARTIFACT_DIR };
|
|
78
131
|
|
|
132
|
+
const harborEnvironment =
|
|
133
|
+
value.SELFBENCH_HARBOR_ENVIRONMENT ??
|
|
134
|
+
matchingHarborEnvironment(value.SELFBENCH_EXECUTION_BACKEND) ??
|
|
135
|
+
fail(
|
|
136
|
+
`SELFBENCH_HARBOR_ENVIRONMENT is required for ${value.SELFBENCH_EXECUTION_BACKEND} execution`,
|
|
137
|
+
);
|
|
138
|
+
let execution: ExecutionConfig;
|
|
139
|
+
switch (value.SELFBENCH_EXECUTION_BACKEND) {
|
|
140
|
+
case "docker":
|
|
141
|
+
execution = { kind: "docker", image: value.SELFBENCH_DOCKER_IMAGE };
|
|
142
|
+
break;
|
|
143
|
+
case "modal":
|
|
144
|
+
execution = {
|
|
145
|
+
kind: "modal",
|
|
146
|
+
app: value.SELFBENCH_MODAL_APP,
|
|
147
|
+
image: value.SELFBENCH_MODAL_IMAGE,
|
|
148
|
+
...(value.SELFBENCH_MODAL_ENVIRONMENT
|
|
149
|
+
? { environment: value.SELFBENCH_MODAL_ENVIRONMENT }
|
|
150
|
+
: {}),
|
|
151
|
+
};
|
|
152
|
+
break;
|
|
153
|
+
case "vercel": {
|
|
154
|
+
const image =
|
|
155
|
+
value.SELFBENCH_VERCEL_IMAGE ??
|
|
156
|
+
fail("SELFBENCH_VERCEL_IMAGE is required for Vercel execution");
|
|
157
|
+
if (!isDigestPinnedOciImage(image)) {
|
|
158
|
+
fail("SELFBENCH_VERCEL_IMAGE must be pinned by sha256 digest");
|
|
159
|
+
}
|
|
160
|
+
execution = {
|
|
161
|
+
kind: "vercel",
|
|
162
|
+
image,
|
|
163
|
+
timeoutCapMs: vercelTimeoutCap(value.SELFBENCH_VERCEL_TIMEOUT_CAP),
|
|
164
|
+
};
|
|
165
|
+
break;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
79
169
|
return {
|
|
80
170
|
apiHost: value.SELFBENCH_API_HOST,
|
|
81
171
|
apiPort: value.SELFBENCH_API_PORT,
|
|
@@ -85,20 +175,10 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBe
|
|
|
85
175
|
: {}),
|
|
86
176
|
activityConcurrency:
|
|
87
177
|
value.SELFBENCH_ACTIVITY_CONCURRENCY ??
|
|
88
|
-
|
|
178
|
+
DEFAULT_ACTIVITY_CONCURRENCY[value.SELFBENCH_EXECUTION_BACKEND],
|
|
89
179
|
artifact,
|
|
90
|
-
harborEnvironment
|
|
91
|
-
execution
|
|
92
|
-
value.SELFBENCH_EXECUTION_BACKEND === "modal"
|
|
93
|
-
? {
|
|
94
|
-
kind: "modal",
|
|
95
|
-
app: value.SELFBENCH_MODAL_APP,
|
|
96
|
-
image: value.SELFBENCH_MODAL_IMAGE,
|
|
97
|
-
...(value.SELFBENCH_MODAL_ENVIRONMENT
|
|
98
|
-
? { environment: value.SELFBENCH_MODAL_ENVIRONMENT }
|
|
99
|
-
: {}),
|
|
100
|
-
}
|
|
101
|
-
: { kind: "docker", image: value.SELFBENCH_DOCKER_IMAGE },
|
|
180
|
+
harborEnvironment,
|
|
181
|
+
execution,
|
|
102
182
|
temporal: {
|
|
103
183
|
address: value.SELFBENCH_TEMPORAL_ADDRESS,
|
|
104
184
|
namespace: value.SELFBENCH_TEMPORAL_NAMESPACE,
|
|
@@ -109,6 +189,51 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBe
|
|
|
109
189
|
};
|
|
110
190
|
}
|
|
111
191
|
|
|
192
|
+
export function loadWorkerConfig(
|
|
193
|
+
environment: NodeJS.ProcessEnv = process.env,
|
|
194
|
+
): SelfBenchWorkerConfig {
|
|
195
|
+
const config = loadConfig(environment);
|
|
196
|
+
if (config.execution.kind !== "vercel") {
|
|
197
|
+
return { ...config, execution: config.execution };
|
|
198
|
+
}
|
|
199
|
+
return {
|
|
200
|
+
...config,
|
|
201
|
+
execution: {
|
|
202
|
+
...config.execution,
|
|
203
|
+
credentials: vercelCredentials(environment),
|
|
204
|
+
},
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
export function defaultStandaloneConcurrency(
|
|
209
|
+
config: Pick<SelfBenchConfig, "activityConcurrency" | "execution">,
|
|
210
|
+
existingDefault: number,
|
|
211
|
+
): number {
|
|
212
|
+
return config.execution.kind === "vercel" ? config.activityConcurrency : existingDefault;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function vercelCredentials(environment: NodeJS.ProcessEnv): VercelCredentials {
|
|
216
|
+
const token = environment.VERCEL_TOKEN?.trim();
|
|
217
|
+
const teamId = environment.VERCEL_TEAM_ID?.trim();
|
|
218
|
+
const projectId = environment.VERCEL_PROJECT_ID?.trim();
|
|
219
|
+
if (!token || !teamId || !projectId) {
|
|
220
|
+
fail("VERCEL_TOKEN, VERCEL_TEAM_ID, and VERCEL_PROJECT_ID are required for Vercel execution");
|
|
221
|
+
}
|
|
222
|
+
return { token, teamId, projectId };
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function vercelTimeoutCap(value: string | undefined): number {
|
|
226
|
+
if (value === undefined) {
|
|
227
|
+
return STANDARD_VERCEL_TIMEOUT_CAP_MS;
|
|
228
|
+
}
|
|
229
|
+
return z
|
|
230
|
+
.number()
|
|
231
|
+
.int()
|
|
232
|
+
.min(100)
|
|
233
|
+
.max(STANDARD_VERCEL_TIMEOUT_CAP_MS)
|
|
234
|
+
.parse(parseSandboxTimeoutCapText(value));
|
|
235
|
+
}
|
|
236
|
+
|
|
112
237
|
function fail(message: string): never {
|
|
113
238
|
throw new Error(message);
|
|
114
239
|
}
|
package/src/contracts.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { EXECUTION_BACKENDS, HARBOR_ENVIRONMENTS, matchingHarborEnvironment } from "./providers.js";
|
|
2
3
|
|
|
3
4
|
export const commitSchema = z.string().regex(/^[0-9a-f]{40}$/i, "expected a full commit SHA");
|
|
4
5
|
|
|
@@ -34,6 +35,37 @@ const candidateCountsSchema = z
|
|
|
34
35
|
message: "at most 100 candidates may be requested",
|
|
35
36
|
});
|
|
36
37
|
|
|
38
|
+
const runVersionSchema = z
|
|
39
|
+
.object({
|
|
40
|
+
selfbenchCommit: commitSchema,
|
|
41
|
+
executionBackend: z.enum(EXECUTION_BACKENDS),
|
|
42
|
+
harborEnvironment: z.enum(HARBOR_ENVIRONMENTS).optional(),
|
|
43
|
+
sandboxImage: z.string().min(1),
|
|
44
|
+
sandboxTimeoutCapMs: z.number().int().min(100).optional(),
|
|
45
|
+
schema: z.literal(1),
|
|
46
|
+
})
|
|
47
|
+
.transform((version, context) => {
|
|
48
|
+
const harborEnvironment =
|
|
49
|
+
version.harborEnvironment ?? matchingHarborEnvironment(version.executionBackend);
|
|
50
|
+
if (!harborEnvironment) {
|
|
51
|
+
context.addIssue({
|
|
52
|
+
code: "custom",
|
|
53
|
+
message: `harborEnvironment is required for ${version.executionBackend} execution`,
|
|
54
|
+
path: ["harborEnvironment"],
|
|
55
|
+
});
|
|
56
|
+
return z.NEVER;
|
|
57
|
+
}
|
|
58
|
+
if (version.sandboxTimeoutCapMs !== undefined && version.executionBackend !== "vercel") {
|
|
59
|
+
context.addIssue({
|
|
60
|
+
code: "custom",
|
|
61
|
+
message: "sandboxTimeoutCapMs is only valid for Vercel execution",
|
|
62
|
+
path: ["sandboxTimeoutCapMs"],
|
|
63
|
+
});
|
|
64
|
+
return z.NEVER;
|
|
65
|
+
}
|
|
66
|
+
return { ...version, harborEnvironment };
|
|
67
|
+
});
|
|
68
|
+
|
|
37
69
|
export const runRequestSchema = z.object({
|
|
38
70
|
runId: z.string().regex(/^[a-z0-9][a-z0-9-]{2,62}$/),
|
|
39
71
|
repository: repositoryRefSchema,
|
|
@@ -44,12 +76,7 @@ export const runRequestSchema = z.object({
|
|
|
44
76
|
model: z.string().min(1),
|
|
45
77
|
reasoningEffort: z.literal("high"),
|
|
46
78
|
}),
|
|
47
|
-
version:
|
|
48
|
-
selfbenchCommit: commitSchema,
|
|
49
|
-
executionBackend: z.enum(["docker", "modal"]),
|
|
50
|
-
sandboxImage: z.string().min(1),
|
|
51
|
-
schema: z.literal(1),
|
|
52
|
-
}),
|
|
79
|
+
version: runVersionSchema,
|
|
53
80
|
});
|
|
54
81
|
|
|
55
82
|
export type RunRequest = z.infer<typeof runRequestSchema>;
|
package/src/eval-main.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import { parseArgs } from "node:util";
|
|
4
4
|
import { MATRIX_MODELS, type MatrixModel, runMatrix } from "./evaluate.js";
|
|
5
|
+
import { isHarborEnvironment } from "./providers.js";
|
|
5
6
|
|
|
6
7
|
const parsed = parseArgs({
|
|
7
8
|
options: {
|
|
@@ -33,8 +34,8 @@ Options:
|
|
|
33
34
|
-h, --help Show this help`);
|
|
34
35
|
process.exit(0);
|
|
35
36
|
}
|
|
36
|
-
const environment = parsed.values.environment;
|
|
37
|
-
if (environment
|
|
37
|
+
const environment = parsed.values.environment ?? "modal";
|
|
38
|
+
if (!isHarborEnvironment(environment)) {
|
|
38
39
|
throw new Error("--environment must be docker or modal");
|
|
39
40
|
}
|
|
40
41
|
const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
|
package/src/evaluate.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
2
2
|
import { homedir, tmpdir } from "node:os";
|
|
3
3
|
import { basename, join, resolve } from "node:path";
|
|
4
|
+
import { harborChildEnvironment } from "./harbor-environment.js";
|
|
4
5
|
import { archiveIncompleteHarborJob, tryReadHarborJobResult } from "./harbor-results.js";
|
|
5
6
|
import { parallelMap } from "./parallel.js";
|
|
6
7
|
import { runCommand } from "./process.js";
|
|
8
|
+
import type { HarborEnvironment } from "./providers.js";
|
|
7
9
|
import { assertCodexSubscriptionAuth, openAiApiKey } from "./subscription-auth.js";
|
|
8
10
|
|
|
9
11
|
export const MATRIX_MODELS = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"] as const;
|
|
@@ -15,7 +17,7 @@ export interface MatrixOptions {
|
|
|
15
17
|
readonly tasksPath?: string;
|
|
16
18
|
readonly jobsDirectory: string;
|
|
17
19
|
readonly harborPath?: string;
|
|
18
|
-
readonly environment?:
|
|
20
|
+
readonly environment?: HarborEnvironment;
|
|
19
21
|
readonly concurrency?: number;
|
|
20
22
|
readonly authPath?: string;
|
|
21
23
|
readonly models?: readonly MatrixModel[];
|
|
@@ -114,7 +116,7 @@ async function runTrial(input: {
|
|
|
114
116
|
readonly model: MatrixModel;
|
|
115
117
|
readonly jobsDirectory: string;
|
|
116
118
|
readonly harborPath: string;
|
|
117
|
-
readonly environment:
|
|
119
|
+
readonly environment: HarborEnvironment;
|
|
118
120
|
readonly authPath?: string;
|
|
119
121
|
}): Promise<MatrixTrialSummary> {
|
|
120
122
|
const taskId = basename(input.taskDirectory);
|
|
@@ -125,7 +127,7 @@ async function runTrial(input: {
|
|
|
125
127
|
}
|
|
126
128
|
await archiveIncompleteHarborJob(input.jobsDirectory, jobName);
|
|
127
129
|
|
|
128
|
-
const environment =
|
|
130
|
+
const environment = harborChildEnvironment();
|
|
129
131
|
if (input.authPath) {
|
|
130
132
|
delete environment.OPENAI_API_KEY;
|
|
131
133
|
environment.CODEX_FORCE_AUTH_JSON = "1";
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
const VERCEL_CONTROL_CREDENTIALS = [
|
|
2
|
+
"VERCEL_AUTH_TOKEN",
|
|
3
|
+
"VERCEL_TOKEN",
|
|
4
|
+
"VERCEL_TEAM_ID",
|
|
5
|
+
"VERCEL_PROJECT_ID",
|
|
6
|
+
"VERCEL_OIDC_TOKEN",
|
|
7
|
+
] as const;
|
|
8
|
+
|
|
9
|
+
export function harborChildEnvironment(
|
|
10
|
+
environment: NodeJS.ProcessEnv = process.env,
|
|
11
|
+
): NodeJS.ProcessEnv {
|
|
12
|
+
const child = { ...environment };
|
|
13
|
+
for (const key of VERCEL_CONTROL_CREDENTIALS) {
|
|
14
|
+
delete child[key];
|
|
15
|
+
}
|
|
16
|
+
return child;
|
|
17
|
+
}
|
package/src/harbor-results.ts
CHANGED
|
@@ -20,6 +20,8 @@ const infrastructurePatterns = [
|
|
|
20
20
|
/modal.*(?:unavailable|timed out|timeout)/i,
|
|
21
21
|
/image build for im-[a-z0-9]+ failed/i,
|
|
22
22
|
/all predefined address pools have been fully subnetted/i,
|
|
23
|
+
/mounts denied/i,
|
|
24
|
+
/network-policy[\s\S]*could not process rule/i,
|
|
23
25
|
];
|
|
24
26
|
|
|
25
27
|
class IncompleteHarborJobError extends Error {}
|
package/src/process.ts
CHANGED
|
@@ -6,6 +6,8 @@ export interface CommandResult {
|
|
|
6
6
|
readonly exitCode: number;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
+
export type CommandOutputHandler = (stream: "stdout" | "stderr", chunk: Uint8Array) => void;
|
|
10
|
+
|
|
9
11
|
export interface CommandOptions {
|
|
10
12
|
readonly cwd?: string;
|
|
11
13
|
readonly env?: NodeJS.ProcessEnv;
|
|
@@ -14,7 +16,7 @@ export interface CommandOptions {
|
|
|
14
16
|
readonly inactivityTimeoutMs?: number;
|
|
15
17
|
readonly allowFailure?: boolean;
|
|
16
18
|
readonly signal?: AbortSignal;
|
|
17
|
-
readonly onOutput?:
|
|
19
|
+
readonly onOutput?: CommandOutputHandler;
|
|
18
20
|
}
|
|
19
21
|
|
|
20
22
|
export class InactivityTimeoutError extends Error {
|
package/src/providers.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export const EXECUTION_BACKENDS = ["docker", "modal", "vercel"] as const;
|
|
2
|
+
export type ExecutionBackend = (typeof EXECUTION_BACKENDS)[number];
|
|
3
|
+
|
|
4
|
+
export const HARBOR_ENVIRONMENTS = ["docker", "modal"] as const;
|
|
5
|
+
export type HarborEnvironment = (typeof HARBOR_ENVIRONMENTS)[number];
|
|
6
|
+
|
|
7
|
+
const executionBackends: ReadonlySet<string> = new Set(EXECUTION_BACKENDS);
|
|
8
|
+
const harborEnvironments: ReadonlySet<string> = new Set(HARBOR_ENVIRONMENTS);
|
|
9
|
+
|
|
10
|
+
export function isExecutionBackend(value: string): value is ExecutionBackend {
|
|
11
|
+
return executionBackends.has(value);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function isHarborEnvironment(value: string): value is HarborEnvironment {
|
|
15
|
+
return harborEnvironments.has(value);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function matchingHarborEnvironment(
|
|
19
|
+
backend: ExecutionBackend,
|
|
20
|
+
): HarborEnvironment | undefined {
|
|
21
|
+
return isHarborEnvironment(backend) ? backend : undefined;
|
|
22
|
+
}
|
package/src/reaudit-main.ts
CHANGED
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
couplingReviewInput,
|
|
11
11
|
couplingReviewSchema,
|
|
12
12
|
} from "./codex-review.js";
|
|
13
|
-
import {
|
|
13
|
+
import { defaultStandaloneConcurrency, loadWorkerConfig } from "./config.js";
|
|
14
14
|
import {
|
|
15
15
|
buildCouplingEvidence,
|
|
16
16
|
discoverContractArtifacts,
|
|
@@ -27,7 +27,7 @@ const parsed = parseArgs({
|
|
|
27
27
|
tasks: { type: "string" },
|
|
28
28
|
task: { type: "string" },
|
|
29
29
|
output: { type: "string" },
|
|
30
|
-
concurrency: { type: "string"
|
|
30
|
+
concurrency: { type: "string" },
|
|
31
31
|
help: { type: "boolean", short: "h" },
|
|
32
32
|
},
|
|
33
33
|
strict: true,
|
|
@@ -39,7 +39,7 @@ Usage:
|
|
|
39
39
|
self-bench-reaudit --tasks DIRECTORY --output REPORT.json [options]
|
|
40
40
|
|
|
41
41
|
Options:
|
|
42
|
-
--concurrency N Concurrent Sol reviews (default: 10)
|
|
42
|
+
--concurrency N Concurrent Sol reviews (default: 10; Vercel: worker setting, default 4)
|
|
43
43
|
--task ID Review only one task ID
|
|
44
44
|
-h, --help Show this help`);
|
|
45
45
|
process.exit(0);
|
|
@@ -47,8 +47,11 @@ Options:
|
|
|
47
47
|
|
|
48
48
|
const tasksDirectory = resolve(parsed.values.tasks ?? fail("--tasks is required"));
|
|
49
49
|
const outputPath = resolve(parsed.values.output ?? fail("--output is required"));
|
|
50
|
-
const
|
|
51
|
-
const
|
|
50
|
+
const config = loadWorkerConfig();
|
|
51
|
+
const concurrency = positiveInteger(
|
|
52
|
+
parsed.values.concurrency ?? String(defaultStandaloneConcurrency(config, 10)),
|
|
53
|
+
"--concurrency",
|
|
54
|
+
);
|
|
52
55
|
const sandbox = createSandboxExecutor(config.execution);
|
|
53
56
|
const [taskIds, modelAuth, reviewer] = await Promise.all([
|
|
54
57
|
readdir(tasksDirectory),
|
package/src/repair-main.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { tmpdir } from "node:os";
|
|
|
5
5
|
import { basename, dirname, join, resolve } from "node:path";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
import { parseArgs } from "node:util";
|
|
8
|
-
import {
|
|
8
|
+
import { defaultStandaloneConcurrency, loadWorkerConfig } from "./config.js";
|
|
9
9
|
import { parallelMap } from "./parallel.js";
|
|
10
10
|
import { runCommand } from "./process.js";
|
|
11
11
|
import { createSandboxExecutor } from "./sandbox.js";
|
|
@@ -17,7 +17,7 @@ const parsed = parseArgs({
|
|
|
17
17
|
review: { type: "string" },
|
|
18
18
|
output: { type: "string" },
|
|
19
19
|
task: { type: "string" },
|
|
20
|
-
concurrency: { type: "string"
|
|
20
|
+
concurrency: { type: "string" },
|
|
21
21
|
model: { type: "string", default: "gpt-5.6-sol" },
|
|
22
22
|
help: { type: "boolean", short: "h" },
|
|
23
23
|
},
|
|
@@ -30,7 +30,7 @@ Usage:
|
|
|
30
30
|
self-bench-repair --tasks DIRECTORY --review REPORT.json --output DIRECTORY [options]
|
|
31
31
|
|
|
32
32
|
Options:
|
|
33
|
-
--concurrency N Concurrent repair sandboxes (default: 9)
|
|
33
|
+
--concurrency N Concurrent repair sandboxes (default: 9; Vercel: worker setting, default 4)
|
|
34
34
|
--model MODEL Codex subscription model (default: gpt-5.6-sol)
|
|
35
35
|
--task ID Repair only one coupled task
|
|
36
36
|
-h, --help Show this help`);
|
|
@@ -43,7 +43,11 @@ const outputDirectory = resolve(parsed.values.output ?? fail("--output is requir
|
|
|
43
43
|
if (outputDirectory === tasksDirectory || outputDirectory.startsWith(`${tasksDirectory}/`)) {
|
|
44
44
|
throw new Error("--output must be outside --tasks");
|
|
45
45
|
}
|
|
46
|
-
const
|
|
46
|
+
const config = loadWorkerConfig();
|
|
47
|
+
const concurrency = positiveInteger(
|
|
48
|
+
parsed.values.concurrency ?? String(defaultStandaloneConcurrency(config, 9)),
|
|
49
|
+
"--concurrency",
|
|
50
|
+
);
|
|
47
51
|
const rawReview = JSON.parse(await readFile(reviewPath, "utf8")) as unknown;
|
|
48
52
|
const review = reviewReport(rawReview);
|
|
49
53
|
const coupled = new Map(
|
|
@@ -77,7 +81,6 @@ for (const source of sourceTasks) {
|
|
|
77
81
|
}
|
|
78
82
|
}
|
|
79
83
|
|
|
80
|
-
const config = loadConfig();
|
|
81
84
|
const sandbox = createSandboxExecutor(config.execution);
|
|
82
85
|
const [modelAuth, repairer] = await Promise.all([
|
|
83
86
|
loadCodexModelAuth(),
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
SandboxExecutor,
|
|
3
|
+
SandboxRequest,
|
|
4
|
+
SandboxResult,
|
|
5
|
+
SandboxRunOptions,
|
|
6
|
+
} from "./sandbox.js";
|
|
7
|
+
|
|
8
|
+
export const STANDARD_VERCEL_TIMEOUT_CAP_MS = 2 * 60 * 60 * 1_000;
|
|
9
|
+
export const HOBBY_VERCEL_TIMEOUT_CAP_MS = 45 * 60 * 1_000;
|
|
10
|
+
|
|
11
|
+
export function parseSandboxTimeoutCapText(value: string): number | undefined {
|
|
12
|
+
const normalized = value.trim().toLowerCase();
|
|
13
|
+
if (normalized === "") {
|
|
14
|
+
return undefined;
|
|
15
|
+
}
|
|
16
|
+
const match = /^(\d+)(ms|s|m|h)?$/.exec(normalized);
|
|
17
|
+
if (!match?.[1]) {
|
|
18
|
+
return Number.NaN;
|
|
19
|
+
}
|
|
20
|
+
const amount = Number(match[1]);
|
|
21
|
+
const multiplier =
|
|
22
|
+
match[2] === "h"
|
|
23
|
+
? 60 * 60 * 1_000
|
|
24
|
+
: match[2] === "m"
|
|
25
|
+
? 60 * 1_000
|
|
26
|
+
: match[2] === "s"
|
|
27
|
+
? 1_000
|
|
28
|
+
: 1;
|
|
29
|
+
return amount * multiplier;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export class TimeoutCappedSandboxExecutor implements SandboxExecutor {
|
|
33
|
+
readonly #delegate: SandboxExecutor;
|
|
34
|
+
readonly #timeoutCapMs: number;
|
|
35
|
+
|
|
36
|
+
constructor(delegate: SandboxExecutor, timeoutCapMs: number) {
|
|
37
|
+
if (!Number.isInteger(timeoutCapMs) || timeoutCapMs < 100) {
|
|
38
|
+
throw new Error("sandbox timeout cap must be an integer of at least 100ms");
|
|
39
|
+
}
|
|
40
|
+
this.#delegate = delegate;
|
|
41
|
+
this.#timeoutCapMs = timeoutCapMs;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async run(request: SandboxRequest, options?: SandboxRunOptions): Promise<SandboxResult> {
|
|
45
|
+
return await this.#delegate.run(
|
|
46
|
+
request.timeoutMs > this.#timeoutCapMs
|
|
47
|
+
? { ...request, timeoutMs: this.#timeoutCapMs }
|
|
48
|
+
: request,
|
|
49
|
+
options,
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
close(): void {
|
|
54
|
+
this.#delegate.close();
|
|
55
|
+
}
|
|
56
|
+
}
|
package/src/sandbox.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { SelfBenchWorkerConfig } from "./config.js";
|
|
2
2
|
import { DockerSandboxExecutor } from "./docker-executor.js";
|
|
3
3
|
import { ModalSandboxExecutor } from "./modal-executor.js";
|
|
4
|
+
import { TimeoutCappedSandboxExecutor } from "./sandbox-timeout.js";
|
|
5
|
+
import { VercelSandboxExecutor } from "./vercel-executor.js";
|
|
4
6
|
|
|
5
7
|
export interface SandboxFile {
|
|
6
8
|
readonly path: string;
|
|
@@ -44,8 +46,16 @@ export interface SandboxExecutor {
|
|
|
44
46
|
close(): void;
|
|
45
47
|
}
|
|
46
48
|
|
|
47
|
-
export function createSandboxExecutor(config:
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
49
|
+
export function createSandboxExecutor(config: SelfBenchWorkerConfig["execution"]): SandboxExecutor {
|
|
50
|
+
switch (config.kind) {
|
|
51
|
+
case "docker":
|
|
52
|
+
return new DockerSandboxExecutor(config);
|
|
53
|
+
case "modal":
|
|
54
|
+
return new ModalSandboxExecutor(config);
|
|
55
|
+
case "vercel":
|
|
56
|
+
return new TimeoutCappedSandboxExecutor(
|
|
57
|
+
new VercelSandboxExecutor(config),
|
|
58
|
+
config.timeoutCapMs,
|
|
59
|
+
);
|
|
60
|
+
}
|
|
51
61
|
}
|
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
couplingReviewInput,
|
|
13
13
|
couplingReviewSchema,
|
|
14
14
|
} from "../codex-review.js";
|
|
15
|
-
import type { SelfBenchConfig } from "../config.js";
|
|
15
|
+
import type { SelfBenchConfig, SelfBenchWorkerConfig } from "../config.js";
|
|
16
16
|
import {
|
|
17
17
|
type ArtifactRef,
|
|
18
18
|
type AuditResult,
|
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
scanBaseContractArtifacts,
|
|
35
35
|
} from "../coupling.js";
|
|
36
36
|
import { assertPullRequestBelongsToRepository } from "../github.js";
|
|
37
|
+
import { harborChildEnvironment } from "../harbor-environment.js";
|
|
37
38
|
import {
|
|
38
39
|
type HarborJobResult,
|
|
39
40
|
harborInfrastructureError,
|
|
@@ -114,7 +115,7 @@ export interface SelfBenchActivities {
|
|
|
114
115
|
buildExport(input: ExportInput): Promise<ArtifactRef>;
|
|
115
116
|
}
|
|
116
117
|
|
|
117
|
-
export function createActivities(config:
|
|
118
|
+
export function createActivities(config: SelfBenchWorkerConfig): SelfBenchActivities {
|
|
118
119
|
const store = createArtifactStore(config.artifact);
|
|
119
120
|
const sandbox = createSandboxExecutor(config.execution);
|
|
120
121
|
return {
|
|
@@ -1054,7 +1055,12 @@ async function runHarborGate(
|
|
|
1054
1055
|
"--yes",
|
|
1055
1056
|
"--quiet",
|
|
1056
1057
|
],
|
|
1057
|
-
{
|
|
1058
|
+
{
|
|
1059
|
+
allowFailure: true,
|
|
1060
|
+
env: harborChildEnvironment(),
|
|
1061
|
+
timeoutMs: 3 * 60 * 60 * 1000,
|
|
1062
|
+
signal,
|
|
1063
|
+
},
|
|
1058
1064
|
);
|
|
1059
1065
|
if (result.exitCode !== 0) {
|
|
1060
1066
|
throw ApplicationFailure.create({
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { fileURLToPath } from "node:url";
|
|
2
2
|
import { Worker } from "@temporalio/worker";
|
|
3
|
-
import {
|
|
3
|
+
import { loadWorkerConfig } from "../config.js";
|
|
4
4
|
import { removeEmptyModalCredentialOverrides } from "../modal-auth.js";
|
|
5
5
|
import { runCommand } from "../process.js";
|
|
6
6
|
import { createActivities } from "./activities.js";
|
|
7
7
|
import { connectTemporalWorker } from "./connection.js";
|
|
8
8
|
|
|
9
9
|
removeEmptyModalCredentialOverrides();
|
|
10
|
-
const config =
|
|
10
|
+
const config = loadWorkerConfig();
|
|
11
11
|
if (config.execution.kind === "docker" || config.harborEnvironment === "docker") {
|
|
12
12
|
await runCommand("docker", ["info"], { timeoutMs: 30_000 });
|
|
13
13
|
await runCommand("docker", ["compose", "version"], { timeoutMs: 30_000 });
|