self-bench 0.3.3 → 0.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/Dockerfile.sandbox +1 -1
  2. package/README.md +20 -0
  3. package/bun.lock +48 -0
  4. package/compose.yaml +7 -1
  5. package/dist/agent-smoke-main.js +3 -2
  6. package/dist/agent-smoke-main.js.map +1 -1
  7. package/dist/agent-smoke.d.ts +2 -1
  8. package/dist/agent-smoke.d.ts.map +1 -1
  9. package/dist/agent-smoke.js +6 -1
  10. package/dist/agent-smoke.js.map +1 -1
  11. package/dist/api.d.ts +24 -0
  12. package/dist/api.d.ts.map +1 -1
  13. package/dist/api.js +25 -18
  14. package/dist/api.js.map +1 -1
  15. package/dist/build-metadata.js +1 -1
  16. package/dist/cli.js +67 -11
  17. package/dist/cli.js.map +1 -1
  18. package/dist/config.d.ts +36 -10
  19. package/dist/config.d.ts.map +1 -1
  20. package/dist/config.js +89 -15
  21. package/dist/config.js.map +1 -1
  22. package/dist/contracts.d.ts +22 -2
  23. package/dist/contracts.d.ts.map +1 -1
  24. package/dist/contracts.js +31 -6
  25. package/dist/contracts.js.map +1 -1
  26. package/dist/eval-main.js +3 -2
  27. package/dist/eval-main.js.map +1 -1
  28. package/dist/evaluate.d.ts +2 -1
  29. package/dist/evaluate.d.ts.map +1 -1
  30. package/dist/evaluate.js +2 -1
  31. package/dist/evaluate.js.map +1 -1
  32. package/dist/harbor-environment.d.ts +2 -0
  33. package/dist/harbor-environment.d.ts.map +1 -0
  34. package/dist/harbor-environment.js +15 -0
  35. package/dist/harbor-environment.js.map +1 -0
  36. package/dist/harbor-results.d.ts.map +1 -1
  37. package/dist/harbor-results.js +2 -0
  38. package/dist/harbor-results.js.map +1 -1
  39. package/dist/process.d.ts +2 -1
  40. package/dist/process.d.ts.map +1 -1
  41. package/dist/process.js.map +1 -1
  42. package/dist/providers.d.ts +8 -0
  43. package/dist/providers.d.ts.map +1 -0
  44. package/dist/providers.js +14 -0
  45. package/dist/providers.js.map +1 -0
  46. package/dist/reaudit-main.js +5 -5
  47. package/dist/reaudit-main.js.map +1 -1
  48. package/dist/repair-main.js +5 -5
  49. package/dist/repair-main.js.map +1 -1
  50. package/dist/sandbox-author.bundle.js +40 -6
  51. package/dist/sandbox-timeout.d.ts +11 -0
  52. package/dist/sandbox-timeout.d.ts.map +1 -0
  53. package/dist/sandbox-timeout.js +41 -0
  54. package/dist/sandbox-timeout.js.map +1 -0
  55. package/dist/sandbox-validation-repair.bundle.js +40 -6
  56. package/dist/sandbox.d.ts +2 -2
  57. package/dist/sandbox.d.ts.map +1 -1
  58. package/dist/sandbox.js +10 -3
  59. package/dist/sandbox.js.map +1 -1
  60. package/dist/temporal/activities.d.ts +2 -2
  61. package/dist/temporal/activities.d.ts.map +1 -1
  62. package/dist/temporal/activities.js +7 -1
  63. package/dist/temporal/activities.js.map +1 -1
  64. package/dist/temporal/worker-main.js +2 -2
  65. package/dist/temporal/worker-main.js.map +1 -1
  66. package/dist/terminal-prompts.d.ts +26 -0
  67. package/dist/terminal-prompts.d.ts.map +1 -0
  68. package/dist/terminal-prompts.js +137 -0
  69. package/dist/terminal-prompts.js.map +1 -0
  70. package/dist/terminal-reporter.d.ts +30 -0
  71. package/dist/terminal-reporter.d.ts.map +1 -0
  72. package/dist/terminal-reporter.js +90 -0
  73. package/dist/terminal-reporter.js.map +1 -0
  74. package/dist/validate-main.js +8 -2
  75. package/dist/validate-main.js.map +1 -1
  76. package/dist/vercel-cli.d.ts +64 -0
  77. package/dist/vercel-cli.d.ts.map +1 -0
  78. package/dist/vercel-cli.js +303 -0
  79. package/dist/vercel-cli.js.map +1 -0
  80. package/dist/vercel-command.d.ts +14 -0
  81. package/dist/vercel-command.d.ts.map +1 -0
  82. package/dist/vercel-command.js +93 -0
  83. package/dist/vercel-command.js.map +1 -0
  84. package/dist/vercel-executor.d.ts +14 -0
  85. package/dist/vercel-executor.d.ts.map +1 -0
  86. package/dist/vercel-executor.js +356 -0
  87. package/dist/vercel-executor.js.map +1 -0
  88. package/dist/vercel-fetch.d.ts +6 -0
  89. package/dist/vercel-fetch.d.ts.map +1 -0
  90. package/dist/vercel-fetch.js +39 -0
  91. package/dist/vercel-fetch.js.map +1 -0
  92. package/dist/vercel-profile.d.ts +34 -0
  93. package/dist/vercel-profile.d.ts.map +1 -0
  94. package/dist/vercel-profile.js +244 -0
  95. package/dist/vercel-profile.js.map +1 -0
  96. package/dist/vercel-runtime-image.d.ts +51 -0
  97. package/dist/vercel-runtime-image.d.ts.map +1 -0
  98. package/dist/vercel-runtime-image.js +123 -0
  99. package/dist/vercel-runtime-image.js.map +1 -0
  100. package/dist/vercel-setup-probe.d.ts +49 -0
  101. package/dist/vercel-setup-probe.d.ts.map +1 -0
  102. package/dist/vercel-setup-probe.js +151 -0
  103. package/dist/vercel-setup-probe.js.map +1 -0
  104. package/dist/vercel-setup.d.ts +42 -0
  105. package/dist/vercel-setup.d.ts.map +1 -0
  106. package/dist/vercel-setup.js +280 -0
  107. package/dist/vercel-setup.js.map +1 -0
  108. package/docs/evaluations.md +2 -0
  109. package/docs/operations.md +128 -6
  110. package/package.json +5 -1
  111. package/review/index.html +14 -0
  112. package/review/src/App.tsx +338 -0
  113. package/review/src/export-loader.test.ts +57 -0
  114. package/review/src/export-loader.ts +151 -0
  115. package/review/src/main.tsx +15 -0
  116. package/review/src/styles.css +162 -0
  117. package/review/src/types.ts +47 -0
  118. package/review/tsconfig.json +16 -0
  119. package/review/vite.config.ts +20 -0
  120. package/scripts/verify-package.ts +27 -3
  121. package/src/agent-smoke-main.ts +3 -2
  122. package/src/agent-smoke.ts +8 -2
  123. package/src/api.ts +31 -18
  124. package/src/cli.ts +72 -11
  125. package/src/config.ts +150 -25
  126. package/src/contracts.ts +33 -6
  127. package/src/eval-main.ts +3 -2
  128. package/src/evaluate.ts +5 -3
  129. package/src/harbor-environment.ts +17 -0
  130. package/src/harbor-results.ts +2 -0
  131. package/src/process.ts +3 -1
  132. package/src/providers.ts +22 -0
  133. package/src/reaudit-main.ts +8 -5
  134. package/src/repair-main.ts +8 -5
  135. package/src/sandbox-timeout.ts +56 -0
  136. package/src/sandbox.ts +15 -5
  137. package/src/temporal/activities.ts +9 -3
  138. package/src/temporal/worker-main.ts +2 -2
  139. package/src/terminal-prompts.ts +197 -0
  140. package/src/terminal-reporter.ts +126 -0
  141. package/src/validate-main.ts +9 -3
  142. package/src/vercel-cli.ts +403 -0
  143. package/src/vercel-command.ts +127 -0
  144. package/src/vercel-executor.ts +448 -0
  145. package/src/vercel-fetch.ts +45 -0
  146. package/src/vercel-profile.ts +319 -0
  147. package/src/vercel-runtime-image.ts +204 -0
  148. package/src/vercel-setup-probe.ts +229 -0
  149. package/src/vercel-setup.ts +419 -0
package/src/config.ts CHANGED
@@ -1,4 +1,15 @@
1
1
  import { z } from "zod";
2
+ import {
3
+ EXECUTION_BACKENDS,
4
+ type ExecutionBackend,
5
+ HARBOR_ENVIRONMENTS,
6
+ type HarborEnvironment,
7
+ matchingHarborEnvironment,
8
+ } from "./providers.js";
9
+ import { parseSandboxTimeoutCapText, STANDARD_VERCEL_TIMEOUT_CAP_MS } from "./sandbox-timeout.js";
10
+
11
+ const emptyStringAsUndefined = (value: unknown): unknown =>
12
+ typeof value === "string" && value.trim() === "" ? undefined : value;
2
13
 
3
14
  const environmentSchema = z.object({
4
15
  SELFBENCH_API_HOST: z.string().default("127.0.0.1"),
@@ -8,12 +19,19 @@ const environmentSchema = z.object({
8
19
  SELFBENCH_ARTIFACT_DIR: z.string().default(".selfbench/artifacts"),
9
20
  SELFBENCH_GCS_BUCKET: z.string().optional(),
10
21
  SELFBENCH_GCS_PREFIX: z.string().default("selfbench"),
11
- SELFBENCH_EXECUTION_BACKEND: z.enum(["docker", "modal"]).default("docker"),
22
+ SELFBENCH_EXECUTION_BACKEND: z.enum(EXECUTION_BACKENDS).default("docker"),
12
23
  SELFBENCH_DOCKER_IMAGE: z.string().default("selfbench-sandbox:local"),
13
- SELFBENCH_HARBOR_ENVIRONMENT: z.enum(["docker", "modal"]).optional(),
24
+ SELFBENCH_HARBOR_ENVIRONMENT: z.preprocess(
25
+ emptyStringAsUndefined,
26
+ z.enum(HARBOR_ENVIRONMENTS).optional(),
27
+ ),
14
28
  SELFBENCH_MODAL_APP: z.string().default("selfbench"),
15
29
  SELFBENCH_MODAL_ENVIRONMENT: z.string().optional(),
16
30
  SELFBENCH_MODAL_IMAGE: z.string().default("node:22-bookworm"),
31
+ SELFBENCH_VERCEL_IMAGE: z.preprocess(emptyStringAsUndefined, z.string().trim().min(1).optional()),
32
+ // Keep provider-specific validation in the Vercel branch so a stale Vercel
33
+ // variable cannot break an otherwise unrelated Docker or Modal worker.
34
+ SELFBENCH_VERCEL_TIMEOUT_CAP: z.preprocess(emptyStringAsUndefined, z.string().optional()),
17
35
  SELFBENCH_BUILD_COMMIT: z.preprocess(
18
36
  (value) => (value === "" ? undefined : value),
19
37
  z
@@ -21,7 +39,10 @@ const environmentSchema = z.object({
21
39
  .regex(/^[0-9a-f]{40}$/i)
22
40
  .optional(),
23
41
  ),
24
- SELFBENCH_ACTIVITY_CONCURRENCY: z.coerce.number().int().min(1).max(100).optional(),
42
+ SELFBENCH_ACTIVITY_CONCURRENCY: z.preprocess(
43
+ emptyStringAsUndefined,
44
+ z.coerce.number().int().min(1).max(100).optional(),
45
+ ),
25
46
  SELFBENCH_TASK_QUEUE: z.string().default("selfbench-dev"),
26
47
  SELFBENCH_TEMPORAL_ADDRESS: z.string().default("127.0.0.1:7233"),
27
48
  SELFBENCH_TEMPORAL_NAMESPACE: z.string().default("default"),
@@ -32,6 +53,37 @@ const environmentSchema = z.object({
32
53
  .default(false),
33
54
  });
34
55
 
56
+ export interface VercelCredentials {
57
+ readonly token: string;
58
+ readonly teamId: string;
59
+ readonly projectId: string;
60
+ }
61
+
62
+ type ExecutionConfig =
63
+ | { readonly kind: "docker"; readonly image: string }
64
+ | {
65
+ readonly kind: "modal";
66
+ readonly app: string;
67
+ readonly environment?: string;
68
+ readonly image: string;
69
+ }
70
+ | { readonly kind: "vercel"; readonly image: string; readonly timeoutCapMs: number };
71
+
72
+ type WorkerExecutionConfig =
73
+ | Exclude<ExecutionConfig, { readonly kind: "vercel" }>
74
+ | {
75
+ readonly kind: "vercel";
76
+ readonly image: string;
77
+ readonly timeoutCapMs: number;
78
+ readonly credentials: VercelCredentials;
79
+ };
80
+
81
+ const DEFAULT_ACTIVITY_CONCURRENCY = {
82
+ docker: 1,
83
+ modal: 20,
84
+ vercel: 4,
85
+ } as const satisfies Record<ExecutionBackend, number>;
86
+
35
87
  export interface SelfBenchConfig {
36
88
  readonly apiHost: string;
37
89
  readonly apiPort: number;
@@ -41,15 +93,8 @@ export interface SelfBenchConfig {
41
93
  readonly artifact:
42
94
  | { readonly kind: "local"; readonly directory: string }
43
95
  | { readonly kind: "gcs"; readonly bucket: string; readonly prefix: string };
44
- readonly execution:
45
- | { readonly kind: "docker"; readonly image: string }
46
- | {
47
- readonly kind: "modal";
48
- readonly app: string;
49
- readonly environment?: string;
50
- readonly image: string;
51
- };
52
- readonly harborEnvironment: "docker" | "modal";
96
+ readonly execution: ExecutionConfig;
97
+ readonly harborEnvironment: HarborEnvironment;
53
98
  readonly temporal: {
54
99
  readonly address: string;
55
100
  readonly namespace: string;
@@ -59,6 +104,14 @@ export interface SelfBenchConfig {
59
104
  };
60
105
  }
61
106
 
107
+ export interface SelfBenchWorkerConfig extends Omit<SelfBenchConfig, "execution"> {
108
+ readonly execution: WorkerExecutionConfig;
109
+ }
110
+
111
+ export function isDigestPinnedOciImage(image: string): boolean {
112
+ return /^[^@\s]+@sha256:[0-9a-f]{64}$/i.test(image);
113
+ }
114
+
62
115
  export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBenchConfig {
63
116
  const value = environmentSchema.parse(environment);
64
117
  if (
@@ -76,6 +129,43 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBe
76
129
  }
77
130
  : { kind: "local" as const, directory: value.SELFBENCH_ARTIFACT_DIR };
78
131
 
132
+ const harborEnvironment =
133
+ value.SELFBENCH_HARBOR_ENVIRONMENT ??
134
+ matchingHarborEnvironment(value.SELFBENCH_EXECUTION_BACKEND) ??
135
+ fail(
136
+ `SELFBENCH_HARBOR_ENVIRONMENT is required for ${value.SELFBENCH_EXECUTION_BACKEND} execution`,
137
+ );
138
+ let execution: ExecutionConfig;
139
+ switch (value.SELFBENCH_EXECUTION_BACKEND) {
140
+ case "docker":
141
+ execution = { kind: "docker", image: value.SELFBENCH_DOCKER_IMAGE };
142
+ break;
143
+ case "modal":
144
+ execution = {
145
+ kind: "modal",
146
+ app: value.SELFBENCH_MODAL_APP,
147
+ image: value.SELFBENCH_MODAL_IMAGE,
148
+ ...(value.SELFBENCH_MODAL_ENVIRONMENT
149
+ ? { environment: value.SELFBENCH_MODAL_ENVIRONMENT }
150
+ : {}),
151
+ };
152
+ break;
153
+ case "vercel": {
154
+ const image =
155
+ value.SELFBENCH_VERCEL_IMAGE ??
156
+ fail("SELFBENCH_VERCEL_IMAGE is required for Vercel execution");
157
+ if (!isDigestPinnedOciImage(image)) {
158
+ fail("SELFBENCH_VERCEL_IMAGE must be pinned by sha256 digest");
159
+ }
160
+ execution = {
161
+ kind: "vercel",
162
+ image,
163
+ timeoutCapMs: vercelTimeoutCap(value.SELFBENCH_VERCEL_TIMEOUT_CAP),
164
+ };
165
+ break;
166
+ }
167
+ }
168
+
79
169
  return {
80
170
  apiHost: value.SELFBENCH_API_HOST,
81
171
  apiPort: value.SELFBENCH_API_PORT,
@@ -85,20 +175,10 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBe
85
175
  : {}),
86
176
  activityConcurrency:
87
177
  value.SELFBENCH_ACTIVITY_CONCURRENCY ??
88
- (value.SELFBENCH_EXECUTION_BACKEND === "modal" ? 20 : 1),
178
+ DEFAULT_ACTIVITY_CONCURRENCY[value.SELFBENCH_EXECUTION_BACKEND],
89
179
  artifact,
90
- harborEnvironment: value.SELFBENCH_HARBOR_ENVIRONMENT ?? value.SELFBENCH_EXECUTION_BACKEND,
91
- execution:
92
- value.SELFBENCH_EXECUTION_BACKEND === "modal"
93
- ? {
94
- kind: "modal",
95
- app: value.SELFBENCH_MODAL_APP,
96
- image: value.SELFBENCH_MODAL_IMAGE,
97
- ...(value.SELFBENCH_MODAL_ENVIRONMENT
98
- ? { environment: value.SELFBENCH_MODAL_ENVIRONMENT }
99
- : {}),
100
- }
101
- : { kind: "docker", image: value.SELFBENCH_DOCKER_IMAGE },
180
+ harborEnvironment,
181
+ execution,
102
182
  temporal: {
103
183
  address: value.SELFBENCH_TEMPORAL_ADDRESS,
104
184
  namespace: value.SELFBENCH_TEMPORAL_NAMESPACE,
@@ -109,6 +189,51 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBe
109
189
  };
110
190
  }
111
191
 
192
+ export function loadWorkerConfig(
193
+ environment: NodeJS.ProcessEnv = process.env,
194
+ ): SelfBenchWorkerConfig {
195
+ const config = loadConfig(environment);
196
+ if (config.execution.kind !== "vercel") {
197
+ return { ...config, execution: config.execution };
198
+ }
199
+ return {
200
+ ...config,
201
+ execution: {
202
+ ...config.execution,
203
+ credentials: vercelCredentials(environment),
204
+ },
205
+ };
206
+ }
207
+
208
+ export function defaultStandaloneConcurrency(
209
+ config: Pick<SelfBenchConfig, "activityConcurrency" | "execution">,
210
+ existingDefault: number,
211
+ ): number {
212
+ return config.execution.kind === "vercel" ? config.activityConcurrency : existingDefault;
213
+ }
214
+
215
+ function vercelCredentials(environment: NodeJS.ProcessEnv): VercelCredentials {
216
+ const token = environment.VERCEL_TOKEN?.trim();
217
+ const teamId = environment.VERCEL_TEAM_ID?.trim();
218
+ const projectId = environment.VERCEL_PROJECT_ID?.trim();
219
+ if (!token || !teamId || !projectId) {
220
+ fail("VERCEL_TOKEN, VERCEL_TEAM_ID, and VERCEL_PROJECT_ID are required for Vercel execution");
221
+ }
222
+ return { token, teamId, projectId };
223
+ }
224
+
225
+ function vercelTimeoutCap(value: string | undefined): number {
226
+ if (value === undefined) {
227
+ return STANDARD_VERCEL_TIMEOUT_CAP_MS;
228
+ }
229
+ return z
230
+ .number()
231
+ .int()
232
+ .min(100)
233
+ .max(STANDARD_VERCEL_TIMEOUT_CAP_MS)
234
+ .parse(parseSandboxTimeoutCapText(value));
235
+ }
236
+
112
237
  function fail(message: string): never {
113
238
  throw new Error(message);
114
239
  }
package/src/contracts.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { z } from "zod";
2
+ import { EXECUTION_BACKENDS, HARBOR_ENVIRONMENTS, matchingHarborEnvironment } from "./providers.js";
2
3
 
3
4
  export const commitSchema = z.string().regex(/^[0-9a-f]{40}$/i, "expected a full commit SHA");
4
5
 
@@ -34,6 +35,37 @@ const candidateCountsSchema = z
34
35
  message: "at most 100 candidates may be requested",
35
36
  });
36
37
 
38
+ const runVersionSchema = z
39
+ .object({
40
+ selfbenchCommit: commitSchema,
41
+ executionBackend: z.enum(EXECUTION_BACKENDS),
42
+ harborEnvironment: z.enum(HARBOR_ENVIRONMENTS).optional(),
43
+ sandboxImage: z.string().min(1),
44
+ sandboxTimeoutCapMs: z.number().int().min(100).optional(),
45
+ schema: z.literal(1),
46
+ })
47
+ .transform((version, context) => {
48
+ const harborEnvironment =
49
+ version.harborEnvironment ?? matchingHarborEnvironment(version.executionBackend);
50
+ if (!harborEnvironment) {
51
+ context.addIssue({
52
+ code: "custom",
53
+ message: `harborEnvironment is required for ${version.executionBackend} execution`,
54
+ path: ["harborEnvironment"],
55
+ });
56
+ return z.NEVER;
57
+ }
58
+ if (version.sandboxTimeoutCapMs !== undefined && version.executionBackend !== "vercel") {
59
+ context.addIssue({
60
+ code: "custom",
61
+ message: "sandboxTimeoutCapMs is only valid for Vercel execution",
62
+ path: ["sandboxTimeoutCapMs"],
63
+ });
64
+ return z.NEVER;
65
+ }
66
+ return { ...version, harborEnvironment };
67
+ });
68
+
37
69
  export const runRequestSchema = z.object({
38
70
  runId: z.string().regex(/^[a-z0-9][a-z0-9-]{2,62}$/),
39
71
  repository: repositoryRefSchema,
@@ -44,12 +76,7 @@ export const runRequestSchema = z.object({
44
76
  model: z.string().min(1),
45
77
  reasoningEffort: z.literal("high"),
46
78
  }),
47
- version: z.object({
48
- selfbenchCommit: commitSchema,
49
- executionBackend: z.enum(["docker", "modal"]),
50
- sandboxImage: z.string().min(1),
51
- schema: z.literal(1),
52
- }),
79
+ version: runVersionSchema,
53
80
  });
54
81
 
55
82
  export type RunRequest = z.infer<typeof runRequestSchema>;
package/src/eval-main.ts CHANGED
@@ -2,6 +2,7 @@
2
2
 
3
3
  import { parseArgs } from "node:util";
4
4
  import { MATRIX_MODELS, type MatrixModel, runMatrix } from "./evaluate.js";
5
+ import { isHarborEnvironment } from "./providers.js";
5
6
 
6
7
  const parsed = parseArgs({
7
8
  options: {
@@ -33,8 +34,8 @@ Options:
33
34
  -h, --help Show this help`);
34
35
  process.exit(0);
35
36
  }
36
- const environment = parsed.values.environment;
37
- if (environment !== "docker" && environment !== "modal") {
37
+ const environment = parsed.values.environment ?? "modal";
38
+ if (!isHarborEnvironment(environment)) {
38
39
  throw new Error("--environment must be docker or modal");
39
40
  }
40
41
  const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
package/src/evaluate.ts CHANGED
@@ -1,9 +1,11 @@
1
1
  import { cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
2
2
  import { homedir, tmpdir } from "node:os";
3
3
  import { basename, join, resolve } from "node:path";
4
+ import { harborChildEnvironment } from "./harbor-environment.js";
4
5
  import { archiveIncompleteHarborJob, tryReadHarborJobResult } from "./harbor-results.js";
5
6
  import { parallelMap } from "./parallel.js";
6
7
  import { runCommand } from "./process.js";
8
+ import type { HarborEnvironment } from "./providers.js";
7
9
  import { assertCodexSubscriptionAuth, openAiApiKey } from "./subscription-auth.js";
8
10
 
9
11
  export const MATRIX_MODELS = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"] as const;
@@ -15,7 +17,7 @@ export interface MatrixOptions {
15
17
  readonly tasksPath?: string;
16
18
  readonly jobsDirectory: string;
17
19
  readonly harborPath?: string;
18
- readonly environment?: "docker" | "modal";
20
+ readonly environment?: HarborEnvironment;
19
21
  readonly concurrency?: number;
20
22
  readonly authPath?: string;
21
23
  readonly models?: readonly MatrixModel[];
@@ -114,7 +116,7 @@ async function runTrial(input: {
114
116
  readonly model: MatrixModel;
115
117
  readonly jobsDirectory: string;
116
118
  readonly harborPath: string;
117
- readonly environment: "docker" | "modal";
119
+ readonly environment: HarborEnvironment;
118
120
  readonly authPath?: string;
119
121
  }): Promise<MatrixTrialSummary> {
120
122
  const taskId = basename(input.taskDirectory);
@@ -125,7 +127,7 @@ async function runTrial(input: {
125
127
  }
126
128
  await archiveIncompleteHarborJob(input.jobsDirectory, jobName);
127
129
 
128
- const environment = { ...process.env };
130
+ const environment = harborChildEnvironment();
129
131
  if (input.authPath) {
130
132
  delete environment.OPENAI_API_KEY;
131
133
  environment.CODEX_FORCE_AUTH_JSON = "1";
@@ -0,0 +1,17 @@
1
+ const VERCEL_CONTROL_CREDENTIALS = [
2
+ "VERCEL_AUTH_TOKEN",
3
+ "VERCEL_TOKEN",
4
+ "VERCEL_TEAM_ID",
5
+ "VERCEL_PROJECT_ID",
6
+ "VERCEL_OIDC_TOKEN",
7
+ ] as const;
8
+
9
+ export function harborChildEnvironment(
10
+ environment: NodeJS.ProcessEnv = process.env,
11
+ ): NodeJS.ProcessEnv {
12
+ const child = { ...environment };
13
+ for (const key of VERCEL_CONTROL_CREDENTIALS) {
14
+ delete child[key];
15
+ }
16
+ return child;
17
+ }
@@ -20,6 +20,8 @@ const infrastructurePatterns = [
20
20
  /modal.*(?:unavailable|timed out|timeout)/i,
21
21
  /image build for im-[a-z0-9]+ failed/i,
22
22
  /all predefined address pools have been fully subnetted/i,
23
+ /mounts denied/i,
24
+ /network-policy[\s\S]*could not process rule/i,
23
25
  ];
24
26
 
25
27
  class IncompleteHarborJobError extends Error {}
package/src/process.ts CHANGED
@@ -6,6 +6,8 @@ export interface CommandResult {
6
6
  readonly exitCode: number;
7
7
  }
8
8
 
9
+ export type CommandOutputHandler = (stream: "stdout" | "stderr", chunk: Uint8Array) => void;
10
+
9
11
  export interface CommandOptions {
10
12
  readonly cwd?: string;
11
13
  readonly env?: NodeJS.ProcessEnv;
@@ -14,7 +16,7 @@ export interface CommandOptions {
14
16
  readonly inactivityTimeoutMs?: number;
15
17
  readonly allowFailure?: boolean;
16
18
  readonly signal?: AbortSignal;
17
- readonly onOutput?: (stream: "stdout" | "stderr", chunk: Uint8Array) => void;
19
+ readonly onOutput?: CommandOutputHandler;
18
20
  }
19
21
 
20
22
  export class InactivityTimeoutError extends Error {
@@ -0,0 +1,22 @@
1
+ export const EXECUTION_BACKENDS = ["docker", "modal", "vercel"] as const;
2
+ export type ExecutionBackend = (typeof EXECUTION_BACKENDS)[number];
3
+
4
+ export const HARBOR_ENVIRONMENTS = ["docker", "modal"] as const;
5
+ export type HarborEnvironment = (typeof HARBOR_ENVIRONMENTS)[number];
6
+
7
+ const executionBackends: ReadonlySet<string> = new Set(EXECUTION_BACKENDS);
8
+ const harborEnvironments: ReadonlySet<string> = new Set(HARBOR_ENVIRONMENTS);
9
+
10
+ export function isExecutionBackend(value: string): value is ExecutionBackend {
11
+ return executionBackends.has(value);
12
+ }
13
+
14
+ export function isHarborEnvironment(value: string): value is HarborEnvironment {
15
+ return harborEnvironments.has(value);
16
+ }
17
+
18
+ export function matchingHarborEnvironment(
19
+ backend: ExecutionBackend,
20
+ ): HarborEnvironment | undefined {
21
+ return isHarborEnvironment(backend) ? backend : undefined;
22
+ }
@@ -10,7 +10,7 @@ import {
10
10
  couplingReviewInput,
11
11
  couplingReviewSchema,
12
12
  } from "./codex-review.js";
13
- import { loadConfig } from "./config.js";
13
+ import { defaultStandaloneConcurrency, loadWorkerConfig } from "./config.js";
14
14
  import {
15
15
  buildCouplingEvidence,
16
16
  discoverContractArtifacts,
@@ -27,7 +27,7 @@ const parsed = parseArgs({
27
27
  tasks: { type: "string" },
28
28
  task: { type: "string" },
29
29
  output: { type: "string" },
30
- concurrency: { type: "string", default: "10" },
30
+ concurrency: { type: "string" },
31
31
  help: { type: "boolean", short: "h" },
32
32
  },
33
33
  strict: true,
@@ -39,7 +39,7 @@ Usage:
39
39
  self-bench-reaudit --tasks DIRECTORY --output REPORT.json [options]
40
40
 
41
41
  Options:
42
- --concurrency N Concurrent Sol reviews (default: 10)
42
+ --concurrency N Concurrent Sol reviews (default: 10; Vercel: worker setting, default 4)
43
43
  --task ID Review only one task ID
44
44
  -h, --help Show this help`);
45
45
  process.exit(0);
@@ -47,8 +47,11 @@ Options:
47
47
 
48
48
  const tasksDirectory = resolve(parsed.values.tasks ?? fail("--tasks is required"));
49
49
  const outputPath = resolve(parsed.values.output ?? fail("--output is required"));
50
- const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
51
- const config = loadConfig();
50
+ const config = loadWorkerConfig();
51
+ const concurrency = positiveInteger(
52
+ parsed.values.concurrency ?? String(defaultStandaloneConcurrency(config, 10)),
53
+ "--concurrency",
54
+ );
52
55
  const sandbox = createSandboxExecutor(config.execution);
53
56
  const [taskIds, modelAuth, reviewer] = await Promise.all([
54
57
  readdir(tasksDirectory),
@@ -5,7 +5,7 @@ import { tmpdir } from "node:os";
5
5
  import { basename, dirname, join, resolve } from "node:path";
6
6
  import { fileURLToPath } from "node:url";
7
7
  import { parseArgs } from "node:util";
8
- import { loadConfig } from "./config.js";
8
+ import { defaultStandaloneConcurrency, loadWorkerConfig } from "./config.js";
9
9
  import { parallelMap } from "./parallel.js";
10
10
  import { runCommand } from "./process.js";
11
11
  import { createSandboxExecutor } from "./sandbox.js";
@@ -17,7 +17,7 @@ const parsed = parseArgs({
17
17
  review: { type: "string" },
18
18
  output: { type: "string" },
19
19
  task: { type: "string" },
20
- concurrency: { type: "string", default: "9" },
20
+ concurrency: { type: "string" },
21
21
  model: { type: "string", default: "gpt-5.6-sol" },
22
22
  help: { type: "boolean", short: "h" },
23
23
  },
@@ -30,7 +30,7 @@ Usage:
30
30
  self-bench-repair --tasks DIRECTORY --review REPORT.json --output DIRECTORY [options]
31
31
 
32
32
  Options:
33
- --concurrency N Concurrent repair sandboxes (default: 9)
33
+ --concurrency N Concurrent repair sandboxes (default: 9; Vercel: worker setting, default 4)
34
34
  --model MODEL Codex subscription model (default: gpt-5.6-sol)
35
35
  --task ID Repair only one coupled task
36
36
  -h, --help Show this help`);
@@ -43,7 +43,11 @@ const outputDirectory = resolve(parsed.values.output ?? fail("--output is requir
43
43
  if (outputDirectory === tasksDirectory || outputDirectory.startsWith(`${tasksDirectory}/`)) {
44
44
  throw new Error("--output must be outside --tasks");
45
45
  }
46
- const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
46
+ const config = loadWorkerConfig();
47
+ const concurrency = positiveInteger(
48
+ parsed.values.concurrency ?? String(defaultStandaloneConcurrency(config, 9)),
49
+ "--concurrency",
50
+ );
47
51
  const rawReview = JSON.parse(await readFile(reviewPath, "utf8")) as unknown;
48
52
  const review = reviewReport(rawReview);
49
53
  const coupled = new Map(
@@ -77,7 +81,6 @@ for (const source of sourceTasks) {
77
81
  }
78
82
  }
79
83
 
80
- const config = loadConfig();
81
84
  const sandbox = createSandboxExecutor(config.execution);
82
85
  const [modelAuth, repairer] = await Promise.all([
83
86
  loadCodexModelAuth(),
@@ -0,0 +1,56 @@
1
+ import type {
2
+ SandboxExecutor,
3
+ SandboxRequest,
4
+ SandboxResult,
5
+ SandboxRunOptions,
6
+ } from "./sandbox.js";
7
+
8
+ export const STANDARD_VERCEL_TIMEOUT_CAP_MS = 2 * 60 * 60 * 1_000;
9
+ export const HOBBY_VERCEL_TIMEOUT_CAP_MS = 45 * 60 * 1_000;
10
+
11
+ export function parseSandboxTimeoutCapText(value: string): number | undefined {
12
+ const normalized = value.trim().toLowerCase();
13
+ if (normalized === "") {
14
+ return undefined;
15
+ }
16
+ const match = /^(\d+)(ms|s|m|h)?$/.exec(normalized);
17
+ if (!match?.[1]) {
18
+ return Number.NaN;
19
+ }
20
+ const amount = Number(match[1]);
21
+ const multiplier =
22
+ match[2] === "h"
23
+ ? 60 * 60 * 1_000
24
+ : match[2] === "m"
25
+ ? 60 * 1_000
26
+ : match[2] === "s"
27
+ ? 1_000
28
+ : 1;
29
+ return amount * multiplier;
30
+ }
31
+
32
+ export class TimeoutCappedSandboxExecutor implements SandboxExecutor {
33
+ readonly #delegate: SandboxExecutor;
34
+ readonly #timeoutCapMs: number;
35
+
36
+ constructor(delegate: SandboxExecutor, timeoutCapMs: number) {
37
+ if (!Number.isInteger(timeoutCapMs) || timeoutCapMs < 100) {
38
+ throw new Error("sandbox timeout cap must be an integer of at least 100ms");
39
+ }
40
+ this.#delegate = delegate;
41
+ this.#timeoutCapMs = timeoutCapMs;
42
+ }
43
+
44
+ async run(request: SandboxRequest, options?: SandboxRunOptions): Promise<SandboxResult> {
45
+ return await this.#delegate.run(
46
+ request.timeoutMs > this.#timeoutCapMs
47
+ ? { ...request, timeoutMs: this.#timeoutCapMs }
48
+ : request,
49
+ options,
50
+ );
51
+ }
52
+
53
+ close(): void {
54
+ this.#delegate.close();
55
+ }
56
+ }
package/src/sandbox.ts CHANGED
@@ -1,6 +1,8 @@
1
- import type { SelfBenchConfig } from "./config.js";
1
+ import type { SelfBenchWorkerConfig } from "./config.js";
2
2
  import { DockerSandboxExecutor } from "./docker-executor.js";
3
3
  import { ModalSandboxExecutor } from "./modal-executor.js";
4
+ import { TimeoutCappedSandboxExecutor } from "./sandbox-timeout.js";
5
+ import { VercelSandboxExecutor } from "./vercel-executor.js";
4
6
 
5
7
  export interface SandboxFile {
6
8
  readonly path: string;
@@ -44,8 +46,16 @@ export interface SandboxExecutor {
44
46
  close(): void;
45
47
  }
46
48
 
47
- export function createSandboxExecutor(config: SelfBenchConfig["execution"]): SandboxExecutor {
48
- return config.kind === "modal"
49
- ? new ModalSandboxExecutor(config)
50
- : new DockerSandboxExecutor(config);
49
+ export function createSandboxExecutor(config: SelfBenchWorkerConfig["execution"]): SandboxExecutor {
50
+ switch (config.kind) {
51
+ case "docker":
52
+ return new DockerSandboxExecutor(config);
53
+ case "modal":
54
+ return new ModalSandboxExecutor(config);
55
+ case "vercel":
56
+ return new TimeoutCappedSandboxExecutor(
57
+ new VercelSandboxExecutor(config),
58
+ config.timeoutCapMs,
59
+ );
60
+ }
51
61
  }
@@ -12,7 +12,7 @@ import {
12
12
  couplingReviewInput,
13
13
  couplingReviewSchema,
14
14
  } from "../codex-review.js";
15
- import type { SelfBenchConfig } from "../config.js";
15
+ import type { SelfBenchConfig, SelfBenchWorkerConfig } from "../config.js";
16
16
  import {
17
17
  type ArtifactRef,
18
18
  type AuditResult,
@@ -34,6 +34,7 @@ import {
34
34
  scanBaseContractArtifacts,
35
35
  } from "../coupling.js";
36
36
  import { assertPullRequestBelongsToRepository } from "../github.js";
37
+ import { harborChildEnvironment } from "../harbor-environment.js";
37
38
  import {
38
39
  type HarborJobResult,
39
40
  harborInfrastructureError,
@@ -114,7 +115,7 @@ export interface SelfBenchActivities {
114
115
  buildExport(input: ExportInput): Promise<ArtifactRef>;
115
116
  }
116
117
 
117
- export function createActivities(config: SelfBenchConfig): SelfBenchActivities {
118
+ export function createActivities(config: SelfBenchWorkerConfig): SelfBenchActivities {
118
119
  const store = createArtifactStore(config.artifact);
119
120
  const sandbox = createSandboxExecutor(config.execution);
120
121
  return {
@@ -1054,7 +1055,12 @@ async function runHarborGate(
1054
1055
  "--yes",
1055
1056
  "--quiet",
1056
1057
  ],
1057
- { allowFailure: true, timeoutMs: 3 * 60 * 60 * 1000, signal },
1058
+ {
1059
+ allowFailure: true,
1060
+ env: harborChildEnvironment(),
1061
+ timeoutMs: 3 * 60 * 60 * 1000,
1062
+ signal,
1063
+ },
1058
1064
  );
1059
1065
  if (result.exitCode !== 0) {
1060
1066
  throw ApplicationFailure.create({
@@ -1,13 +1,13 @@
1
1
  import { fileURLToPath } from "node:url";
2
2
  import { Worker } from "@temporalio/worker";
3
- import { loadConfig } from "../config.js";
3
+ import { loadWorkerConfig } from "../config.js";
4
4
  import { removeEmptyModalCredentialOverrides } from "../modal-auth.js";
5
5
  import { runCommand } from "../process.js";
6
6
  import { createActivities } from "./activities.js";
7
7
  import { connectTemporalWorker } from "./connection.js";
8
8
 
9
9
  removeEmptyModalCredentialOverrides();
10
- const config = loadConfig();
10
+ const config = loadWorkerConfig();
11
11
  if (config.execution.kind === "docker" || config.harborEnvironment === "docker") {
12
12
  await runCommand("docker", ["info"], { timeoutMs: 30_000 });
13
13
  await runCommand("docker", ["compose", "version"], { timeoutMs: 30_000 });