self-bench 0.3.3 → 0.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/Dockerfile.sandbox +1 -1
  2. package/README.md +20 -0
  3. package/bun.lock +48 -0
  4. package/compose.yaml +7 -1
  5. package/dist/agent-smoke-main.js +3 -2
  6. package/dist/agent-smoke-main.js.map +1 -1
  7. package/dist/agent-smoke.d.ts +2 -1
  8. package/dist/agent-smoke.d.ts.map +1 -1
  9. package/dist/agent-smoke.js +6 -1
  10. package/dist/agent-smoke.js.map +1 -1
  11. package/dist/api.d.ts +24 -0
  12. package/dist/api.d.ts.map +1 -1
  13. package/dist/api.js +25 -18
  14. package/dist/api.js.map +1 -1
  15. package/dist/build-metadata.js +1 -1
  16. package/dist/cli.js +67 -11
  17. package/dist/cli.js.map +1 -1
  18. package/dist/config.d.ts +36 -10
  19. package/dist/config.d.ts.map +1 -1
  20. package/dist/config.js +89 -15
  21. package/dist/config.js.map +1 -1
  22. package/dist/contracts.d.ts +22 -2
  23. package/dist/contracts.d.ts.map +1 -1
  24. package/dist/contracts.js +31 -6
  25. package/dist/contracts.js.map +1 -1
  26. package/dist/eval-main.js +3 -2
  27. package/dist/eval-main.js.map +1 -1
  28. package/dist/evaluate.d.ts +2 -1
  29. package/dist/evaluate.d.ts.map +1 -1
  30. package/dist/evaluate.js +2 -1
  31. package/dist/evaluate.js.map +1 -1
  32. package/dist/harbor-environment.d.ts +2 -0
  33. package/dist/harbor-environment.d.ts.map +1 -0
  34. package/dist/harbor-environment.js +15 -0
  35. package/dist/harbor-environment.js.map +1 -0
  36. package/dist/harbor-results.d.ts.map +1 -1
  37. package/dist/harbor-results.js +2 -0
  38. package/dist/harbor-results.js.map +1 -1
  39. package/dist/process.d.ts +2 -1
  40. package/dist/process.d.ts.map +1 -1
  41. package/dist/process.js.map +1 -1
  42. package/dist/providers.d.ts +8 -0
  43. package/dist/providers.d.ts.map +1 -0
  44. package/dist/providers.js +14 -0
  45. package/dist/providers.js.map +1 -0
  46. package/dist/reaudit-main.js +5 -5
  47. package/dist/reaudit-main.js.map +1 -1
  48. package/dist/repair-main.js +5 -5
  49. package/dist/repair-main.js.map +1 -1
  50. package/dist/sandbox-author.bundle.js +40 -6
  51. package/dist/sandbox-timeout.d.ts +11 -0
  52. package/dist/sandbox-timeout.d.ts.map +1 -0
  53. package/dist/sandbox-timeout.js +41 -0
  54. package/dist/sandbox-timeout.js.map +1 -0
  55. package/dist/sandbox-validation-repair.bundle.js +40 -6
  56. package/dist/sandbox.d.ts +2 -2
  57. package/dist/sandbox.d.ts.map +1 -1
  58. package/dist/sandbox.js +10 -3
  59. package/dist/sandbox.js.map +1 -1
  60. package/dist/temporal/activities.d.ts +2 -2
  61. package/dist/temporal/activities.d.ts.map +1 -1
  62. package/dist/temporal/activities.js +7 -1
  63. package/dist/temporal/activities.js.map +1 -1
  64. package/dist/temporal/worker-main.js +2 -2
  65. package/dist/temporal/worker-main.js.map +1 -1
  66. package/dist/terminal-prompts.d.ts +26 -0
  67. package/dist/terminal-prompts.d.ts.map +1 -0
  68. package/dist/terminal-prompts.js +137 -0
  69. package/dist/terminal-prompts.js.map +1 -0
  70. package/dist/terminal-reporter.d.ts +30 -0
  71. package/dist/terminal-reporter.d.ts.map +1 -0
  72. package/dist/terminal-reporter.js +90 -0
  73. package/dist/terminal-reporter.js.map +1 -0
  74. package/dist/validate-main.js +8 -2
  75. package/dist/validate-main.js.map +1 -1
  76. package/dist/vercel-cli.d.ts +64 -0
  77. package/dist/vercel-cli.d.ts.map +1 -0
  78. package/dist/vercel-cli.js +303 -0
  79. package/dist/vercel-cli.js.map +1 -0
  80. package/dist/vercel-command.d.ts +14 -0
  81. package/dist/vercel-command.d.ts.map +1 -0
  82. package/dist/vercel-command.js +93 -0
  83. package/dist/vercel-command.js.map +1 -0
  84. package/dist/vercel-executor.d.ts +14 -0
  85. package/dist/vercel-executor.d.ts.map +1 -0
  86. package/dist/vercel-executor.js +356 -0
  87. package/dist/vercel-executor.js.map +1 -0
  88. package/dist/vercel-fetch.d.ts +6 -0
  89. package/dist/vercel-fetch.d.ts.map +1 -0
  90. package/dist/vercel-fetch.js +39 -0
  91. package/dist/vercel-fetch.js.map +1 -0
  92. package/dist/vercel-profile.d.ts +34 -0
  93. package/dist/vercel-profile.d.ts.map +1 -0
  94. package/dist/vercel-profile.js +244 -0
  95. package/dist/vercel-profile.js.map +1 -0
  96. package/dist/vercel-runtime-image.d.ts +51 -0
  97. package/dist/vercel-runtime-image.d.ts.map +1 -0
  98. package/dist/vercel-runtime-image.js +123 -0
  99. package/dist/vercel-runtime-image.js.map +1 -0
  100. package/dist/vercel-setup-probe.d.ts +49 -0
  101. package/dist/vercel-setup-probe.d.ts.map +1 -0
  102. package/dist/vercel-setup-probe.js +151 -0
  103. package/dist/vercel-setup-probe.js.map +1 -0
  104. package/dist/vercel-setup.d.ts +42 -0
  105. package/dist/vercel-setup.d.ts.map +1 -0
  106. package/dist/vercel-setup.js +280 -0
  107. package/dist/vercel-setup.js.map +1 -0
  108. package/docs/evaluations.md +2 -0
  109. package/docs/operations.md +128 -6
  110. package/package.json +3 -1
  111. package/src/agent-smoke-main.ts +3 -2
  112. package/src/agent-smoke.ts +8 -2
  113. package/src/api.ts +31 -18
  114. package/src/cli.ts +72 -11
  115. package/src/config.ts +150 -25
  116. package/src/contracts.ts +33 -6
  117. package/src/eval-main.ts +3 -2
  118. package/src/evaluate.ts +5 -3
  119. package/src/harbor-environment.ts +17 -0
  120. package/src/harbor-results.ts +2 -0
  121. package/src/process.ts +3 -1
  122. package/src/providers.ts +22 -0
  123. package/src/reaudit-main.ts +8 -5
  124. package/src/repair-main.ts +8 -5
  125. package/src/sandbox-timeout.ts +56 -0
  126. package/src/sandbox.ts +15 -5
  127. package/src/temporal/activities.ts +9 -3
  128. package/src/temporal/worker-main.ts +2 -2
  129. package/src/terminal-prompts.ts +197 -0
  130. package/src/terminal-reporter.ts +126 -0
  131. package/src/validate-main.ts +9 -3
  132. package/src/vercel-cli.ts +403 -0
  133. package/src/vercel-command.ts +127 -0
  134. package/src/vercel-executor.ts +448 -0
  135. package/src/vercel-fetch.ts +45 -0
  136. package/src/vercel-profile.ts +319 -0
  137. package/src/vercel-runtime-image.ts +204 -0
  138. package/src/vercel-setup-probe.ts +229 -0
  139. package/src/vercel-setup.ts +419 -0
package/src/api.ts CHANGED
@@ -11,6 +11,7 @@ import type { SelfBenchConfig } from "./config.js";
11
11
  import {
12
12
  artifactRefSchema,
13
13
  type RunPhase,
14
+ type RunRequest,
14
15
  type RunStatus,
15
16
  repositoryRefSchema,
16
17
  runRequestSchema,
@@ -68,26 +69,10 @@ export async function startApi(config: SelfBenchConfig): Promise<() => Promise<v
68
69
  return;
69
70
  }
70
71
  if (request.method === "POST" && url.pathname === "/v1/runs") {
71
- const submission = submissionSchema.parse(
72
+ const workflowInput = buildRunRequest(
73
+ config,
72
74
  JSON.parse((await readBody(request)).toString("utf8")),
73
75
  );
74
- const workflowInput = runRequestSchema.parse({
75
- runId: submission.runId,
76
- repository: submission.repository,
77
- provenance: submission.provenance,
78
- candidateCounts: submission.candidateCounts,
79
- authoring: {
80
- provider: "openai-codex",
81
- model: submission.authoringModel,
82
- reasoningEffort: "high",
83
- },
84
- version: {
85
- selfbenchCommit: config.buildCommit ?? submission.selfbenchCommit,
86
- executionBackend: config.execution.kind,
87
- sandboxImage: config.execution.image,
88
- schema: 1,
89
- },
90
- });
91
76
  await client.workflow.start(selfBenchRunWorkflow, {
92
77
  workflowId: workflowInput.runId,
93
78
  taskQueue: config.temporal.taskQueue,
@@ -167,6 +152,34 @@ export async function startApi(config: SelfBenchConfig): Promise<() => Promise<v
167
152
  };
168
153
  }
169
154
 
155
+ export function buildRunRequest(
156
+ config: SelfBenchConfig,
157
+ submission: z.input<typeof submissionSchema>,
158
+ ): RunRequest {
159
+ const parsed = submissionSchema.parse(submission);
160
+ return runRequestSchema.parse({
161
+ runId: parsed.runId,
162
+ repository: parsed.repository,
163
+ provenance: parsed.provenance,
164
+ candidateCounts: parsed.candidateCounts,
165
+ authoring: {
166
+ provider: "openai-codex",
167
+ model: parsed.authoringModel,
168
+ reasoningEffort: "high",
169
+ },
170
+ version: {
171
+ selfbenchCommit: config.buildCommit ?? parsed.selfbenchCommit,
172
+ executionBackend: config.execution.kind,
173
+ harborEnvironment: config.harborEnvironment,
174
+ sandboxImage: config.execution.image,
175
+ ...(config.execution.kind === "vercel"
176
+ ? { sandboxTimeoutCapMs: config.execution.timeoutCapMs }
177
+ : {}),
178
+ schema: 1,
179
+ },
180
+ });
181
+ }
182
+
170
183
  async function queryStatus(
171
184
  handle: ReturnType<Client["workflow"]["getHandle"]>,
172
185
  ): Promise<RunStatus | object> {
package/src/cli.ts CHANGED
@@ -10,12 +10,20 @@ import type { ReadableStream as NodeReadableStream } from "node:stream/web";
10
10
  import { fileURLToPath } from "node:url";
11
11
  import { parseArgs } from "node:util";
12
12
  import { buildCommit } from "./build-metadata.js";
13
+ import { loadWorkerConfig } from "./config.js";
13
14
  import { runCommand } from "./process.js";
14
15
  import { collectGitHubPullRequestProvenance, collectRepositoryProvenance } from "./provenance.js";
16
+ import { isExecutionBackend, isHarborEnvironment, matchingHarborEnvironment } from "./providers.js";
15
17
  import { type PolledRunStatus, waitForRun } from "./run-wait.js";
18
+ import { SetupCanceledError } from "./terminal-prompts.js";
19
+ import { applyVercelProfile } from "./vercel-profile.js";
20
+ import { setupVercel } from "./vercel-setup.js";
16
21
 
17
22
  const [command, ...rest] = process.argv.slice(2);
18
23
  switch (command) {
24
+ case "setup":
25
+ await setup(rest);
26
+ break;
19
27
  case "up":
20
28
  await up(rest);
21
29
  break;
@@ -47,31 +55,74 @@ switch (command) {
47
55
  throw new Error(`unknown command: ${command}`);
48
56
  }
49
57
 
58
+ async function setup(args: string[]): Promise<void> {
59
+ const [provider, ...providerArgs] = args;
60
+ if (provider !== "vercel") {
61
+ fail("setup currently supports only: self-bench setup vercel");
62
+ }
63
+ const parsed = parseArgs({
64
+ args: providerArgs,
65
+ options: {
66
+ profile: { type: "string", default: "default" },
67
+ verbose: { type: "boolean", default: false },
68
+ },
69
+ strict: true,
70
+ });
71
+ try {
72
+ await setupVercel({
73
+ profileName: parsed.values.profile,
74
+ verbose: parsed.values.verbose,
75
+ });
76
+ } catch (error) {
77
+ if (error instanceof SetupCanceledError) {
78
+ process.exitCode = 130;
79
+ return;
80
+ }
81
+ throw error;
82
+ }
83
+ }
84
+
50
85
  async function up(args: string[]): Promise<void> {
51
86
  const parsed = parseArgs({
52
87
  args,
53
88
  options: {
54
89
  backend: { type: "string", default: "docker" },
90
+ "harbor-environment": { type: "string" },
55
91
  "modal-config": { type: "string" },
92
+ "vercel-profile": { type: "string" },
56
93
  },
57
94
  strict: true,
58
95
  });
59
- if (parsed.values.backend !== "docker" && parsed.values.backend !== "modal") {
60
- fail('--backend must be "docker" or "modal"');
96
+ if (!isExecutionBackend(parsed.values.backend)) {
97
+ fail('--backend must be "docker", "modal", or "vercel"');
61
98
  }
62
- if (parsed.values.backend === "docker" && parsed.values["modal-config"] !== undefined) {
63
- fail("--modal-config requires --backend modal");
99
+ if (
100
+ parsed.values["harbor-environment"] !== undefined &&
101
+ !isHarborEnvironment(parsed.values["harbor-environment"])
102
+ ) {
103
+ fail('--harbor-environment must be "docker" or "modal"');
64
104
  }
65
105
 
66
106
  const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
67
107
  const composeFile = resolve(projectRoot, "compose.yaml");
68
108
  const backend = parsed.values.backend;
69
- const environment = {
109
+ const harborEnvironment =
110
+ parsed.values["harbor-environment"] ??
111
+ matchingHarborEnvironment(backend) ??
112
+ fail(`--harbor-environment is required with --backend ${backend}`);
113
+ const usesModal = backend === "modal" || harborEnvironment === "modal";
114
+ if (!usesModal && parsed.values["modal-config"] !== undefined) {
115
+ fail("--modal-config requires Modal generation or Harbor");
116
+ }
117
+ if (backend !== "vercel" && parsed.values["vercel-profile"] !== undefined) {
118
+ fail("--vercel-profile requires Vercel generation");
119
+ }
120
+ let environment: NodeJS.ProcessEnv = {
70
121
  ...process.env,
71
122
  SELFBENCH_BUILD_COMMIT: await resolveSelfBenchCommit(),
72
123
  SELFBENCH_EXECUTION_BACKEND: backend,
73
- SELFBENCH_HARBOR_ENVIRONMENT: backend,
74
- ...(backend === "modal"
124
+ SELFBENCH_HARBOR_ENVIRONMENT: harborEnvironment,
125
+ ...(usesModal
75
126
  ? {
76
127
  SELFBENCH_MODAL_CONFIG_PATH: resolve(
77
128
  parsed.values["modal-config"] ?? resolve(homedir(), ".modal.toml"),
@@ -79,6 +130,10 @@ async function up(args: string[]): Promise<void> {
79
130
  }
80
131
  : {}),
81
132
  };
133
+ if (backend === "vercel") {
134
+ environment = await applyVercelProfile(environment, parsed.values["vercel-profile"]);
135
+ loadWorkerConfig(environment);
136
+ }
82
137
 
83
138
  if (backend === "docker") {
84
139
  await runCommand(
@@ -97,7 +152,9 @@ async function up(args: string[]): Promise<void> {
97
152
  await runCommand("docker", ["compose", "--file", composeFile, "up", "-d", "--build"], {
98
153
  env: environment,
99
154
  });
100
- console.log(`SelfBench is running with the ${backend} backend at http://127.0.0.1:8080`);
155
+ console.log(
156
+ `SelfBench is running with ${backend} generation and ${harborEnvironment} Harbor at http://127.0.0.1:8080`,
157
+ );
101
158
  }
102
159
 
103
160
  async function down(): Promise<void> {
@@ -340,7 +397,9 @@ function printHelp(): void {
340
397
  console.log(`SelfBench creates durable tiered Harbor evaluations.
341
398
 
342
399
  Usage:
343
- self-bench up [--backend docker|modal] [--modal-config PATH]
400
+ self-bench setup vercel [--profile NAME] [--verbose]
401
+ self-bench up [--backend docker|modal|vercel] [--harbor-environment docker|modal]
402
+ [--modal-config PATH] [--vercel-profile NAME]
344
403
  self-bench down
345
404
  self-bench run --repo PATH [--easy-count N] [--medium-count N] [--hard-count N]
346
405
  [--model MODEL] [--run-id ID] [--wait] [--output OUTPUT.tar.gz]
@@ -349,8 +408,10 @@ Usage:
349
408
  self-bench download RUN_ID OUTPUT.tar.gz
350
409
  self-bench list
351
410
 
352
- The up command starts the local stack and configures both sandbox execution and Harbor validation for
353
- one backend. Modal uses ~/.modal.toml unless --modal-config overrides it.
411
+ The up command starts the local stack. Docker and Modal default Harbor to the matching backend; Vercel
412
+ requires --harbor-environment because Harbor does not support Vercel. Modal generation or Harbor uses
413
+ ~/.modal.toml unless --modal-config overrides it. Run self-bench setup vercel once to create or select a
414
+ project, publish the pinned runtime image, verify access, and save an owner-only local profile.
354
415
 
355
416
  The tier counts are candidate authoring budgets, not accepted-task targets. Rejected candidates are not
356
417
  replaced, and the export contains only accepted tasks. The run command performs only repository metadata
package/src/config.ts CHANGED
@@ -1,4 +1,15 @@
1
1
  import { z } from "zod";
2
+ import {
3
+ EXECUTION_BACKENDS,
4
+ type ExecutionBackend,
5
+ HARBOR_ENVIRONMENTS,
6
+ type HarborEnvironment,
7
+ matchingHarborEnvironment,
8
+ } from "./providers.js";
9
+ import { parseSandboxTimeoutCapText, STANDARD_VERCEL_TIMEOUT_CAP_MS } from "./sandbox-timeout.js";
10
+
11
+ const emptyStringAsUndefined = (value: unknown): unknown =>
12
+ typeof value === "string" && value.trim() === "" ? undefined : value;
2
13
 
3
14
  const environmentSchema = z.object({
4
15
  SELFBENCH_API_HOST: z.string().default("127.0.0.1"),
@@ -8,12 +19,19 @@ const environmentSchema = z.object({
8
19
  SELFBENCH_ARTIFACT_DIR: z.string().default(".selfbench/artifacts"),
9
20
  SELFBENCH_GCS_BUCKET: z.string().optional(),
10
21
  SELFBENCH_GCS_PREFIX: z.string().default("selfbench"),
11
- SELFBENCH_EXECUTION_BACKEND: z.enum(["docker", "modal"]).default("docker"),
22
+ SELFBENCH_EXECUTION_BACKEND: z.enum(EXECUTION_BACKENDS).default("docker"),
12
23
  SELFBENCH_DOCKER_IMAGE: z.string().default("selfbench-sandbox:local"),
13
- SELFBENCH_HARBOR_ENVIRONMENT: z.enum(["docker", "modal"]).optional(),
24
+ SELFBENCH_HARBOR_ENVIRONMENT: z.preprocess(
25
+ emptyStringAsUndefined,
26
+ z.enum(HARBOR_ENVIRONMENTS).optional(),
27
+ ),
14
28
  SELFBENCH_MODAL_APP: z.string().default("selfbench"),
15
29
  SELFBENCH_MODAL_ENVIRONMENT: z.string().optional(),
16
30
  SELFBENCH_MODAL_IMAGE: z.string().default("node:22-bookworm"),
31
+ SELFBENCH_VERCEL_IMAGE: z.preprocess(emptyStringAsUndefined, z.string().trim().min(1).optional()),
32
+ // Keep provider-specific validation in the Vercel branch so a stale Vercel
33
+ // variable cannot break an otherwise unrelated Docker or Modal worker.
34
+ SELFBENCH_VERCEL_TIMEOUT_CAP: z.preprocess(emptyStringAsUndefined, z.string().optional()),
17
35
  SELFBENCH_BUILD_COMMIT: z.preprocess(
18
36
  (value) => (value === "" ? undefined : value),
19
37
  z
@@ -21,7 +39,10 @@ const environmentSchema = z.object({
21
39
  .regex(/^[0-9a-f]{40}$/i)
22
40
  .optional(),
23
41
  ),
24
- SELFBENCH_ACTIVITY_CONCURRENCY: z.coerce.number().int().min(1).max(100).optional(),
42
+ SELFBENCH_ACTIVITY_CONCURRENCY: z.preprocess(
43
+ emptyStringAsUndefined,
44
+ z.coerce.number().int().min(1).max(100).optional(),
45
+ ),
25
46
  SELFBENCH_TASK_QUEUE: z.string().default("selfbench-dev"),
26
47
  SELFBENCH_TEMPORAL_ADDRESS: z.string().default("127.0.0.1:7233"),
27
48
  SELFBENCH_TEMPORAL_NAMESPACE: z.string().default("default"),
@@ -32,6 +53,37 @@ const environmentSchema = z.object({
32
53
  .default(false),
33
54
  });
34
55
 
56
+ export interface VercelCredentials {
57
+ readonly token: string;
58
+ readonly teamId: string;
59
+ readonly projectId: string;
60
+ }
61
+
62
+ type ExecutionConfig =
63
+ | { readonly kind: "docker"; readonly image: string }
64
+ | {
65
+ readonly kind: "modal";
66
+ readonly app: string;
67
+ readonly environment?: string;
68
+ readonly image: string;
69
+ }
70
+ | { readonly kind: "vercel"; readonly image: string; readonly timeoutCapMs: number };
71
+
72
+ type WorkerExecutionConfig =
73
+ | Exclude<ExecutionConfig, { readonly kind: "vercel" }>
74
+ | {
75
+ readonly kind: "vercel";
76
+ readonly image: string;
77
+ readonly timeoutCapMs: number;
78
+ readonly credentials: VercelCredentials;
79
+ };
80
+
81
+ const DEFAULT_ACTIVITY_CONCURRENCY = {
82
+ docker: 1,
83
+ modal: 20,
84
+ vercel: 4,
85
+ } as const satisfies Record<ExecutionBackend, number>;
86
+
35
87
  export interface SelfBenchConfig {
36
88
  readonly apiHost: string;
37
89
  readonly apiPort: number;
@@ -41,15 +93,8 @@ export interface SelfBenchConfig {
41
93
  readonly artifact:
42
94
  | { readonly kind: "local"; readonly directory: string }
43
95
  | { readonly kind: "gcs"; readonly bucket: string; readonly prefix: string };
44
- readonly execution:
45
- | { readonly kind: "docker"; readonly image: string }
46
- | {
47
- readonly kind: "modal";
48
- readonly app: string;
49
- readonly environment?: string;
50
- readonly image: string;
51
- };
52
- readonly harborEnvironment: "docker" | "modal";
96
+ readonly execution: ExecutionConfig;
97
+ readonly harborEnvironment: HarborEnvironment;
53
98
  readonly temporal: {
54
99
  readonly address: string;
55
100
  readonly namespace: string;
@@ -59,6 +104,14 @@ export interface SelfBenchConfig {
59
104
  };
60
105
  }
61
106
 
107
+ export interface SelfBenchWorkerConfig extends Omit<SelfBenchConfig, "execution"> {
108
+ readonly execution: WorkerExecutionConfig;
109
+ }
110
+
111
+ export function isDigestPinnedOciImage(image: string): boolean {
112
+ return /^[^@\s]+@sha256:[0-9a-f]{64}$/i.test(image);
113
+ }
114
+
62
115
  export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBenchConfig {
63
116
  const value = environmentSchema.parse(environment);
64
117
  if (
@@ -76,6 +129,43 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBe
76
129
  }
77
130
  : { kind: "local" as const, directory: value.SELFBENCH_ARTIFACT_DIR };
78
131
 
132
+ const harborEnvironment =
133
+ value.SELFBENCH_HARBOR_ENVIRONMENT ??
134
+ matchingHarborEnvironment(value.SELFBENCH_EXECUTION_BACKEND) ??
135
+ fail(
136
+ `SELFBENCH_HARBOR_ENVIRONMENT is required for ${value.SELFBENCH_EXECUTION_BACKEND} execution`,
137
+ );
138
+ let execution: ExecutionConfig;
139
+ switch (value.SELFBENCH_EXECUTION_BACKEND) {
140
+ case "docker":
141
+ execution = { kind: "docker", image: value.SELFBENCH_DOCKER_IMAGE };
142
+ break;
143
+ case "modal":
144
+ execution = {
145
+ kind: "modal",
146
+ app: value.SELFBENCH_MODAL_APP,
147
+ image: value.SELFBENCH_MODAL_IMAGE,
148
+ ...(value.SELFBENCH_MODAL_ENVIRONMENT
149
+ ? { environment: value.SELFBENCH_MODAL_ENVIRONMENT }
150
+ : {}),
151
+ };
152
+ break;
153
+ case "vercel": {
154
+ const image =
155
+ value.SELFBENCH_VERCEL_IMAGE ??
156
+ fail("SELFBENCH_VERCEL_IMAGE is required for Vercel execution");
157
+ if (!isDigestPinnedOciImage(image)) {
158
+ fail("SELFBENCH_VERCEL_IMAGE must be pinned by sha256 digest");
159
+ }
160
+ execution = {
161
+ kind: "vercel",
162
+ image,
163
+ timeoutCapMs: vercelTimeoutCap(value.SELFBENCH_VERCEL_TIMEOUT_CAP),
164
+ };
165
+ break;
166
+ }
167
+ }
168
+
79
169
  return {
80
170
  apiHost: value.SELFBENCH_API_HOST,
81
171
  apiPort: value.SELFBENCH_API_PORT,
@@ -85,20 +175,10 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBe
85
175
  : {}),
86
176
  activityConcurrency:
87
177
  value.SELFBENCH_ACTIVITY_CONCURRENCY ??
88
- (value.SELFBENCH_EXECUTION_BACKEND === "modal" ? 20 : 1),
178
+ DEFAULT_ACTIVITY_CONCURRENCY[value.SELFBENCH_EXECUTION_BACKEND],
89
179
  artifact,
90
- harborEnvironment: value.SELFBENCH_HARBOR_ENVIRONMENT ?? value.SELFBENCH_EXECUTION_BACKEND,
91
- execution:
92
- value.SELFBENCH_EXECUTION_BACKEND === "modal"
93
- ? {
94
- kind: "modal",
95
- app: value.SELFBENCH_MODAL_APP,
96
- image: value.SELFBENCH_MODAL_IMAGE,
97
- ...(value.SELFBENCH_MODAL_ENVIRONMENT
98
- ? { environment: value.SELFBENCH_MODAL_ENVIRONMENT }
99
- : {}),
100
- }
101
- : { kind: "docker", image: value.SELFBENCH_DOCKER_IMAGE },
180
+ harborEnvironment,
181
+ execution,
102
182
  temporal: {
103
183
  address: value.SELFBENCH_TEMPORAL_ADDRESS,
104
184
  namespace: value.SELFBENCH_TEMPORAL_NAMESPACE,
@@ -109,6 +189,51 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): SelfBe
109
189
  };
110
190
  }
111
191
 
192
+ export function loadWorkerConfig(
193
+ environment: NodeJS.ProcessEnv = process.env,
194
+ ): SelfBenchWorkerConfig {
195
+ const config = loadConfig(environment);
196
+ if (config.execution.kind !== "vercel") {
197
+ return { ...config, execution: config.execution };
198
+ }
199
+ return {
200
+ ...config,
201
+ execution: {
202
+ ...config.execution,
203
+ credentials: vercelCredentials(environment),
204
+ },
205
+ };
206
+ }
207
+
208
+ export function defaultStandaloneConcurrency(
209
+ config: Pick<SelfBenchConfig, "activityConcurrency" | "execution">,
210
+ existingDefault: number,
211
+ ): number {
212
+ return config.execution.kind === "vercel" ? config.activityConcurrency : existingDefault;
213
+ }
214
+
215
+ function vercelCredentials(environment: NodeJS.ProcessEnv): VercelCredentials {
216
+ const token = environment.VERCEL_TOKEN?.trim();
217
+ const teamId = environment.VERCEL_TEAM_ID?.trim();
218
+ const projectId = environment.VERCEL_PROJECT_ID?.trim();
219
+ if (!token || !teamId || !projectId) {
220
+ fail("VERCEL_TOKEN, VERCEL_TEAM_ID, and VERCEL_PROJECT_ID are required for Vercel execution");
221
+ }
222
+ return { token, teamId, projectId };
223
+ }
224
+
225
+ function vercelTimeoutCap(value: string | undefined): number {
226
+ if (value === undefined) {
227
+ return STANDARD_VERCEL_TIMEOUT_CAP_MS;
228
+ }
229
+ return z
230
+ .number()
231
+ .int()
232
+ .min(100)
233
+ .max(STANDARD_VERCEL_TIMEOUT_CAP_MS)
234
+ .parse(parseSandboxTimeoutCapText(value));
235
+ }
236
+
112
237
  function fail(message: string): never {
113
238
  throw new Error(message);
114
239
  }
package/src/contracts.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { z } from "zod";
2
+ import { EXECUTION_BACKENDS, HARBOR_ENVIRONMENTS, matchingHarborEnvironment } from "./providers.js";
2
3
 
3
4
  export const commitSchema = z.string().regex(/^[0-9a-f]{40}$/i, "expected a full commit SHA");
4
5
 
@@ -34,6 +35,37 @@ const candidateCountsSchema = z
34
35
  message: "at most 100 candidates may be requested",
35
36
  });
36
37
 
38
+ const runVersionSchema = z
39
+ .object({
40
+ selfbenchCommit: commitSchema,
41
+ executionBackend: z.enum(EXECUTION_BACKENDS),
42
+ harborEnvironment: z.enum(HARBOR_ENVIRONMENTS).optional(),
43
+ sandboxImage: z.string().min(1),
44
+ sandboxTimeoutCapMs: z.number().int().min(100).optional(),
45
+ schema: z.literal(1),
46
+ })
47
+ .transform((version, context) => {
48
+ const harborEnvironment =
49
+ version.harborEnvironment ?? matchingHarborEnvironment(version.executionBackend);
50
+ if (!harborEnvironment) {
51
+ context.addIssue({
52
+ code: "custom",
53
+ message: `harborEnvironment is required for ${version.executionBackend} execution`,
54
+ path: ["harborEnvironment"],
55
+ });
56
+ return z.NEVER;
57
+ }
58
+ if (version.sandboxTimeoutCapMs !== undefined && version.executionBackend !== "vercel") {
59
+ context.addIssue({
60
+ code: "custom",
61
+ message: "sandboxTimeoutCapMs is only valid for Vercel execution",
62
+ path: ["sandboxTimeoutCapMs"],
63
+ });
64
+ return z.NEVER;
65
+ }
66
+ return { ...version, harborEnvironment };
67
+ });
68
+
37
69
  export const runRequestSchema = z.object({
38
70
  runId: z.string().regex(/^[a-z0-9][a-z0-9-]{2,62}$/),
39
71
  repository: repositoryRefSchema,
@@ -44,12 +76,7 @@ export const runRequestSchema = z.object({
44
76
  model: z.string().min(1),
45
77
  reasoningEffort: z.literal("high"),
46
78
  }),
47
- version: z.object({
48
- selfbenchCommit: commitSchema,
49
- executionBackend: z.enum(["docker", "modal"]),
50
- sandboxImage: z.string().min(1),
51
- schema: z.literal(1),
52
- }),
79
+ version: runVersionSchema,
53
80
  });
54
81
 
55
82
  export type RunRequest = z.infer<typeof runRequestSchema>;
package/src/eval-main.ts CHANGED
@@ -2,6 +2,7 @@
2
2
 
3
3
  import { parseArgs } from "node:util";
4
4
  import { MATRIX_MODELS, type MatrixModel, runMatrix } from "./evaluate.js";
5
+ import { isHarborEnvironment } from "./providers.js";
5
6
 
6
7
  const parsed = parseArgs({
7
8
  options: {
@@ -33,8 +34,8 @@ Options:
33
34
  -h, --help Show this help`);
34
35
  process.exit(0);
35
36
  }
36
- const environment = parsed.values.environment;
37
- if (environment !== "docker" && environment !== "modal") {
37
+ const environment = parsed.values.environment ?? "modal";
38
+ if (!isHarborEnvironment(environment)) {
38
39
  throw new Error("--environment must be docker or modal");
39
40
  }
40
41
  const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
package/src/evaluate.ts CHANGED
@@ -1,9 +1,11 @@
1
1
  import { cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
2
2
  import { homedir, tmpdir } from "node:os";
3
3
  import { basename, join, resolve } from "node:path";
4
+ import { harborChildEnvironment } from "./harbor-environment.js";
4
5
  import { archiveIncompleteHarborJob, tryReadHarborJobResult } from "./harbor-results.js";
5
6
  import { parallelMap } from "./parallel.js";
6
7
  import { runCommand } from "./process.js";
8
+ import type { HarborEnvironment } from "./providers.js";
7
9
  import { assertCodexSubscriptionAuth, openAiApiKey } from "./subscription-auth.js";
8
10
 
9
11
  export const MATRIX_MODELS = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"] as const;
@@ -15,7 +17,7 @@ export interface MatrixOptions {
15
17
  readonly tasksPath?: string;
16
18
  readonly jobsDirectory: string;
17
19
  readonly harborPath?: string;
18
- readonly environment?: "docker" | "modal";
20
+ readonly environment?: HarborEnvironment;
19
21
  readonly concurrency?: number;
20
22
  readonly authPath?: string;
21
23
  readonly models?: readonly MatrixModel[];
@@ -114,7 +116,7 @@ async function runTrial(input: {
114
116
  readonly model: MatrixModel;
115
117
  readonly jobsDirectory: string;
116
118
  readonly harborPath: string;
117
- readonly environment: "docker" | "modal";
119
+ readonly environment: HarborEnvironment;
118
120
  readonly authPath?: string;
119
121
  }): Promise<MatrixTrialSummary> {
120
122
  const taskId = basename(input.taskDirectory);
@@ -125,7 +127,7 @@ async function runTrial(input: {
125
127
  }
126
128
  await archiveIncompleteHarborJob(input.jobsDirectory, jobName);
127
129
 
128
- const environment = { ...process.env };
130
+ const environment = harborChildEnvironment();
129
131
  if (input.authPath) {
130
132
  delete environment.OPENAI_API_KEY;
131
133
  environment.CODEX_FORCE_AUTH_JSON = "1";
@@ -0,0 +1,17 @@
1
+ const VERCEL_CONTROL_CREDENTIALS = [
2
+ "VERCEL_AUTH_TOKEN",
3
+ "VERCEL_TOKEN",
4
+ "VERCEL_TEAM_ID",
5
+ "VERCEL_PROJECT_ID",
6
+ "VERCEL_OIDC_TOKEN",
7
+ ] as const;
8
+
9
+ export function harborChildEnvironment(
10
+ environment: NodeJS.ProcessEnv = process.env,
11
+ ): NodeJS.ProcessEnv {
12
+ const child = { ...environment };
13
+ for (const key of VERCEL_CONTROL_CREDENTIALS) {
14
+ delete child[key];
15
+ }
16
+ return child;
17
+ }
@@ -20,6 +20,8 @@ const infrastructurePatterns = [
20
20
  /modal.*(?:unavailable|timed out|timeout)/i,
21
21
  /image build for im-[a-z0-9]+ failed/i,
22
22
  /all predefined address pools have been fully subnetted/i,
23
+ /mounts denied/i,
24
+ /network-policy[\s\S]*could not process rule/i,
23
25
  ];
24
26
 
25
27
  class IncompleteHarborJobError extends Error {}
package/src/process.ts CHANGED
@@ -6,6 +6,8 @@ export interface CommandResult {
6
6
  readonly exitCode: number;
7
7
  }
8
8
 
9
+ export type CommandOutputHandler = (stream: "stdout" | "stderr", chunk: Uint8Array) => void;
10
+
9
11
  export interface CommandOptions {
10
12
  readonly cwd?: string;
11
13
  readonly env?: NodeJS.ProcessEnv;
@@ -14,7 +16,7 @@ export interface CommandOptions {
14
16
  readonly inactivityTimeoutMs?: number;
15
17
  readonly allowFailure?: boolean;
16
18
  readonly signal?: AbortSignal;
17
- readonly onOutput?: (stream: "stdout" | "stderr", chunk: Uint8Array) => void;
19
+ readonly onOutput?: CommandOutputHandler;
18
20
  }
19
21
 
20
22
  export class InactivityTimeoutError extends Error {
@@ -0,0 +1,22 @@
1
+ export const EXECUTION_BACKENDS = ["docker", "modal", "vercel"] as const;
2
+ export type ExecutionBackend = (typeof EXECUTION_BACKENDS)[number];
3
+
4
+ export const HARBOR_ENVIRONMENTS = ["docker", "modal"] as const;
5
+ export type HarborEnvironment = (typeof HARBOR_ENVIRONMENTS)[number];
6
+
7
+ const executionBackends: ReadonlySet<string> = new Set(EXECUTION_BACKENDS);
8
+ const harborEnvironments: ReadonlySet<string> = new Set(HARBOR_ENVIRONMENTS);
9
+
10
+ export function isExecutionBackend(value: string): value is ExecutionBackend {
11
+ return executionBackends.has(value);
12
+ }
13
+
14
+ export function isHarborEnvironment(value: string): value is HarborEnvironment {
15
+ return harborEnvironments.has(value);
16
+ }
17
+
18
+ export function matchingHarborEnvironment(
19
+ backend: ExecutionBackend,
20
+ ): HarborEnvironment | undefined {
21
+ return isHarborEnvironment(backend) ? backend : undefined;
22
+ }
@@ -10,7 +10,7 @@ import {
10
10
  couplingReviewInput,
11
11
  couplingReviewSchema,
12
12
  } from "./codex-review.js";
13
- import { loadConfig } from "./config.js";
13
+ import { defaultStandaloneConcurrency, loadWorkerConfig } from "./config.js";
14
14
  import {
15
15
  buildCouplingEvidence,
16
16
  discoverContractArtifacts,
@@ -27,7 +27,7 @@ const parsed = parseArgs({
27
27
  tasks: { type: "string" },
28
28
  task: { type: "string" },
29
29
  output: { type: "string" },
30
- concurrency: { type: "string", default: "10" },
30
+ concurrency: { type: "string" },
31
31
  help: { type: "boolean", short: "h" },
32
32
  },
33
33
  strict: true,
@@ -39,7 +39,7 @@ Usage:
39
39
  self-bench-reaudit --tasks DIRECTORY --output REPORT.json [options]
40
40
 
41
41
  Options:
42
- --concurrency N Concurrent Sol reviews (default: 10)
42
+ --concurrency N Concurrent Sol reviews (default: 10; Vercel: worker setting, default 4)
43
43
  --task ID Review only one task ID
44
44
  -h, --help Show this help`);
45
45
  process.exit(0);
@@ -47,8 +47,11 @@ Options:
47
47
 
48
48
  const tasksDirectory = resolve(parsed.values.tasks ?? fail("--tasks is required"));
49
49
  const outputPath = resolve(parsed.values.output ?? fail("--output is required"));
50
- const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
51
- const config = loadConfig();
50
+ const config = loadWorkerConfig();
51
+ const concurrency = positiveInteger(
52
+ parsed.values.concurrency ?? String(defaultStandaloneConcurrency(config, 10)),
53
+ "--concurrency",
54
+ );
52
55
  const sandbox = createSandboxExecutor(config.execution);
53
56
  const [taskIds, modelAuth, reviewer] = await Promise.all([
54
57
  readdir(tasksDirectory),