self-bench 0.3.5 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/README.md +16 -14
  2. package/biome.json +7 -1
  3. package/dist/api.d.ts.map +1 -1
  4. package/dist/api.js +7 -7
  5. package/dist/api.js.map +1 -1
  6. package/dist/build-metadata.js +1 -1
  7. package/dist/cli.js +75 -15
  8. package/dist/cli.js.map +1 -1
  9. package/dist/config.js +1 -1
  10. package/dist/contracts.d.ts +1 -0
  11. package/dist/contracts.d.ts.map +1 -1
  12. package/dist/contracts.js +6 -5
  13. package/dist/contracts.js.map +1 -1
  14. package/dist/extensions/discovery.d.ts.map +1 -1
  15. package/dist/extensions/discovery.js +12 -5
  16. package/dist/extensions/discovery.js.map +1 -1
  17. package/dist/harbor-task.d.ts.map +1 -1
  18. package/dist/harbor-task.js +3 -1
  19. package/dist/harbor-task.js.map +1 -1
  20. package/dist/project-paths.d.ts +2 -0
  21. package/dist/project-paths.d.ts.map +1 -0
  22. package/dist/project-paths.js +15 -0
  23. package/dist/project-paths.js.map +1 -0
  24. package/dist/provenance-associations.d.ts +49 -0
  25. package/dist/provenance-associations.d.ts.map +1 -0
  26. package/dist/provenance-associations.js +220 -0
  27. package/dist/provenance-associations.js.map +1 -0
  28. package/dist/provenance.d.ts +34 -13
  29. package/dist/provenance.d.ts.map +1 -1
  30. package/dist/provenance.js +81 -9
  31. package/dist/provenance.js.map +1 -1
  32. package/dist/reaudit-main.js +3 -3
  33. package/dist/reaudit-main.js.map +1 -1
  34. package/dist/repair-main.js +4 -4
  35. package/dist/repair-main.js.map +1 -1
  36. package/dist/{sandbox.d.ts → sandbox/contracts.d.ts} +5 -3
  37. package/dist/sandbox/contracts.d.ts.map +1 -0
  38. package/dist/sandbox/contracts.js +9 -0
  39. package/dist/sandbox/contracts.js.map +1 -0
  40. package/dist/sandbox/index.d.ts +5 -0
  41. package/dist/sandbox/index.d.ts.map +1 -0
  42. package/dist/{sandbox.js → sandbox/index.js} +6 -5
  43. package/dist/sandbox/index.js.map +1 -0
  44. package/dist/sandbox/programs/author.d.ts +3 -0
  45. package/dist/sandbox/programs/author.d.ts.map +1 -0
  46. package/dist/{sandbox-author.js → sandbox/programs/author.js} +2 -2
  47. package/dist/sandbox/programs/author.js.map +1 -0
  48. package/dist/sandbox/programs/prepare-repair.d.ts +7 -0
  49. package/dist/sandbox/programs/prepare-repair.d.ts.map +1 -0
  50. package/dist/sandbox/programs/prepare-repair.js +30 -0
  51. package/dist/sandbox/programs/prepare-repair.js.map +1 -0
  52. package/dist/sandbox/programs/repair.d.ts +3 -0
  53. package/dist/sandbox/programs/repair.d.ts.map +1 -0
  54. package/dist/{sandbox-repair.js → sandbox/programs/repair.js} +7 -28
  55. package/dist/sandbox/programs/repair.js.map +1 -0
  56. package/dist/sandbox/programs/review.d.ts +2 -0
  57. package/dist/sandbox/programs/review.d.ts.map +1 -0
  58. package/dist/{sandbox-review.js → sandbox/programs/review.js} +2 -2
  59. package/dist/sandbox/programs/review.js.map +1 -0
  60. package/dist/sandbox/programs/validation-repair.d.ts +3 -0
  61. package/dist/sandbox/programs/validation-repair.d.ts.map +1 -0
  62. package/dist/{sandbox-validation-repair.js → sandbox/programs/validation-repair.js} +7 -28
  63. package/dist/sandbox/programs/validation-repair.js.map +1 -0
  64. package/dist/{docker-executor.d.ts → sandbox/providers/docker/executor.d.ts} +3 -3
  65. package/dist/sandbox/providers/docker/executor.d.ts.map +1 -0
  66. package/dist/{docker-executor.js → sandbox/providers/docker/executor.js} +2 -2
  67. package/dist/sandbox/providers/docker/executor.js.map +1 -0
  68. package/dist/{modal-executor.d.ts → sandbox/providers/modal/executor.d.ts} +3 -3
  69. package/dist/sandbox/providers/modal/executor.d.ts.map +1 -0
  70. package/dist/{modal-executor.js → sandbox/providers/modal/executor.js} +50 -19
  71. package/dist/sandbox/providers/modal/executor.js.map +1 -0
  72. package/dist/{vercel-command.d.ts → sandbox/providers/vercel/command.d.ts} +3 -3
  73. package/dist/sandbox/providers/vercel/command.d.ts.map +1 -0
  74. package/dist/{vercel-command.js → sandbox/providers/vercel/command.js} +3 -3
  75. package/dist/sandbox/providers/vercel/command.js.map +1 -0
  76. package/dist/{vercel-executor.d.ts → sandbox/providers/vercel/executor.d.ts} +3 -3
  77. package/dist/sandbox/providers/vercel/executor.d.ts.map +1 -0
  78. package/dist/{vercel-executor.js → sandbox/providers/vercel/executor.js} +5 -5
  79. package/dist/sandbox/providers/vercel/executor.js.map +1 -0
  80. package/dist/{vercel-fetch.d.ts → sandbox/providers/vercel/fetch.d.ts} +1 -1
  81. package/dist/sandbox/providers/vercel/fetch.d.ts.map +1 -0
  82. package/dist/{vercel-fetch.js → sandbox/providers/vercel/fetch.js} +1 -1
  83. package/dist/sandbox/providers/vercel/fetch.js.map +1 -0
  84. package/dist/{sandbox-timeout.d.ts → sandbox/timeout.d.ts} +2 -2
  85. package/dist/sandbox/timeout.d.ts.map +1 -0
  86. package/dist/{sandbox-timeout.js → sandbox/timeout.js} +1 -1
  87. package/dist/sandbox/timeout.js.map +1 -0
  88. package/dist/sandbox-author.bundle.js +11 -8
  89. package/dist/sandbox-repair.bundle.js +43 -35
  90. package/dist/sandbox-review.bundle.js +2 -2
  91. package/dist/sandbox-validation-repair.bundle.js +50 -41
  92. package/dist/{vercel-cli.d.ts → setup/vercel/cli.d.ts} +2 -2
  93. package/dist/setup/vercel/cli.d.ts.map +1 -0
  94. package/dist/{vercel-cli.js → setup/vercel/cli.js} +2 -2
  95. package/dist/setup/vercel/cli.js.map +1 -0
  96. package/dist/setup/vercel/index.d.ts +4 -0
  97. package/dist/setup/vercel/index.d.ts.map +1 -0
  98. package/dist/setup/vercel/index.js +3 -0
  99. package/dist/setup/vercel/index.js.map +1 -0
  100. package/dist/{vercel-setup-probe.d.ts → setup/vercel/probe.d.ts} +2 -2
  101. package/dist/setup/vercel/probe.d.ts.map +1 -0
  102. package/dist/{vercel-setup-probe.js → setup/vercel/probe.js} +2 -2
  103. package/dist/setup/vercel/probe.js.map +1 -0
  104. package/dist/{vercel-profile.d.ts → setup/vercel/profile.d.ts} +1 -1
  105. package/dist/setup/vercel/profile.d.ts.map +1 -0
  106. package/dist/{vercel-profile.js → setup/vercel/profile.js} +2 -2
  107. package/dist/setup/vercel/profile.js.map +1 -0
  108. package/dist/{vercel-runtime-image.d.ts → setup/vercel/runtime-image.d.ts} +5 -5
  109. package/dist/setup/vercel/runtime-image.d.ts.map +1 -0
  110. package/dist/{vercel-runtime-image.js → setup/vercel/runtime-image.js} +2 -2
  111. package/dist/setup/vercel/runtime-image.js.map +1 -0
  112. package/dist/{vercel-setup.d.ts → setup/vercel/setup.d.ts} +8 -8
  113. package/dist/setup/vercel/setup.d.ts.map +1 -0
  114. package/dist/{vercel-setup.js → setup/vercel/setup.js} +8 -8
  115. package/dist/setup/vercel/setup.js.map +1 -0
  116. package/dist/temporal/activities.d.ts +12 -0
  117. package/dist/temporal/activities.d.ts.map +1 -1
  118. package/dist/temporal/activities.js +80 -33
  119. package/dist/temporal/activities.js.map +1 -1
  120. package/dist/temporal/workflow.d.ts +1 -1
  121. package/dist/temporal/workflow.d.ts.map +1 -1
  122. package/dist/temporal/workflow.js +90 -65
  123. package/dist/temporal/workflow.js.map +1 -1
  124. package/docs/operations.md +39 -5
  125. package/docs/task-construction.md +3 -1
  126. package/package.json +5 -5
  127. package/scripts/build-sandbox-programs.ts +22 -0
  128. package/scripts/verify-package.ts +25 -4
  129. package/scripts/write-build-metadata.ts +1 -1
  130. package/src/api.ts +7 -6
  131. package/src/cli.ts +99 -15
  132. package/src/config.ts +1 -1
  133. package/src/contracts.ts +7 -5
  134. package/src/extensions/discovery.ts +14 -5
  135. package/src/harbor-task.ts +6 -1
  136. package/src/project-paths.ts +15 -0
  137. package/src/provenance-associations.ts +311 -0
  138. package/src/provenance.ts +132 -28
  139. package/src/reaudit-main.ts +3 -3
  140. package/src/repair-main.ts +4 -4
  141. package/src/{sandbox.ts → sandbox/contracts.ts} +11 -20
  142. package/src/sandbox/index.ts +23 -0
  143. package/src/{sandbox-author.ts → sandbox/programs/author.ts} +1 -1
  144. package/src/sandbox/programs/prepare-repair.ts +44 -0
  145. package/src/{sandbox-repair.ts → sandbox/programs/repair.ts} +7 -31
  146. package/src/{sandbox-review.ts → sandbox/programs/review.ts} +1 -1
  147. package/src/{sandbox-validation-repair.ts → sandbox/programs/validation-repair.ts} +6 -30
  148. package/src/{docker-executor.ts → sandbox/providers/docker/executor.ts} +3 -3
  149. package/src/{modal-executor.ts → sandbox/providers/modal/executor.ts} +66 -25
  150. package/src/{vercel-command.ts → sandbox/providers/vercel/command.ts} +3 -3
  151. package/src/{vercel-executor.ts → sandbox/providers/vercel/executor.ts} +5 -5
  152. package/src/{sandbox-timeout.ts → sandbox/timeout.ts} +1 -1
  153. package/src/{vercel-cli.ts → setup/vercel/cli.ts} +1 -1
  154. package/src/setup/vercel/index.ts +3 -0
  155. package/src/{vercel-setup-probe.ts → setup/vercel/probe.ts} +5 -2
  156. package/src/{vercel-profile.ts → setup/vercel/profile.ts} +4 -1
  157. package/src/{vercel-runtime-image.ts → setup/vercel/runtime-image.ts} +5 -5
  158. package/src/{vercel-setup.ts → setup/vercel/setup.ts} +8 -8
  159. package/src/temporal/activities.ts +164 -63
  160. package/src/temporal/workflow.ts +120 -71
  161. package/tsconfig.json +1 -1
  162. package/dist/docker-executor.d.ts.map +0 -1
  163. package/dist/docker-executor.js.map +0 -1
  164. package/dist/modal-executor.d.ts.map +0 -1
  165. package/dist/modal-executor.js.map +0 -1
  166. package/dist/sandbox-author.d.ts +0 -3
  167. package/dist/sandbox-author.d.ts.map +0 -1
  168. package/dist/sandbox-author.js.map +0 -1
  169. package/dist/sandbox-repair.d.ts +0 -3
  170. package/dist/sandbox-repair.d.ts.map +0 -1
  171. package/dist/sandbox-repair.js.map +0 -1
  172. package/dist/sandbox-review.d.ts +0 -2
  173. package/dist/sandbox-review.d.ts.map +0 -1
  174. package/dist/sandbox-review.js.map +0 -1
  175. package/dist/sandbox-timeout.d.ts.map +0 -1
  176. package/dist/sandbox-timeout.js.map +0 -1
  177. package/dist/sandbox-validation-repair.d.ts +0 -3
  178. package/dist/sandbox-validation-repair.d.ts.map +0 -1
  179. package/dist/sandbox-validation-repair.js.map +0 -1
  180. package/dist/sandbox.d.ts.map +0 -1
  181. package/dist/sandbox.js.map +0 -1
  182. package/dist/vercel-cli.d.ts.map +0 -1
  183. package/dist/vercel-cli.js.map +0 -1
  184. package/dist/vercel-command.d.ts.map +0 -1
  185. package/dist/vercel-command.js.map +0 -1
  186. package/dist/vercel-executor.d.ts.map +0 -1
  187. package/dist/vercel-executor.js.map +0 -1
  188. package/dist/vercel-fetch.d.ts.map +0 -1
  189. package/dist/vercel-fetch.js.map +0 -1
  190. package/dist/vercel-profile.d.ts.map +0 -1
  191. package/dist/vercel-profile.js.map +0 -1
  192. package/dist/vercel-runtime-image.d.ts.map +0 -1
  193. package/dist/vercel-runtime-image.js.map +0 -1
  194. package/dist/vercel-setup-probe.d.ts.map +0 -1
  195. package/dist/vercel-setup-probe.js.map +0 -1
  196. package/dist/vercel-setup.d.ts.map +0 -1
  197. package/dist/vercel-setup.js.map +0 -1
  198. /package/src/{vercel-fetch.ts → sandbox/providers/vercel/fetch.ts} +0 -0
package/src/cli.ts CHANGED
@@ -11,13 +11,23 @@ import { fileURLToPath } from "node:url";
11
11
  import { parseArgs } from "node:util";
12
12
  import { buildCommit } from "./build-metadata.js";
13
13
  import { loadWorkerConfig } from "./config.js";
14
+ import { MAX_CANDIDATES_PER_RUN } from "./contracts.js";
14
15
  import { runCommand } from "./process.js";
15
- import { collectGitHubPullRequestProvenance, collectRepositoryProvenance } from "./provenance.js";
16
+ import {
17
+ collectRepositoryProvenance,
18
+ collectRepositoryProvenanceWithMetadata,
19
+ } from "./provenance.js";
20
+ import {
21
+ applyProvenanceAssociationManifests,
22
+ associationSessionSummaries,
23
+ createProvenanceAssociationManifest,
24
+ resolveMergedPullRequest,
25
+ writeProvenanceAssociationManifest,
26
+ } from "./provenance-associations.js";
16
27
  import { isExecutionBackend, isHarborEnvironment, matchingHarborEnvironment } from "./providers.js";
17
28
  import { type PolledRunStatus, waitForRun } from "./run-wait.js";
29
+ import { applyVercelProfile, setupVercel } from "./setup/vercel/index.js";
18
30
  import { SetupCanceledError } from "./terminal-prompts.js";
19
- import { applyVercelProfile } from "./vercel-profile.js";
20
- import { setupVercel } from "./vercel-setup.js";
21
31
 
22
32
  const [command, ...rest] = process.argv.slice(2);
23
33
  switch (command) {
@@ -30,6 +40,9 @@ switch (command) {
30
40
  case "run":
31
41
  await run(rest);
32
42
  break;
43
+ case "associate":
44
+ await associate(rest);
45
+ break;
33
46
  case "down":
34
47
  await down();
35
48
  break;
@@ -162,6 +175,57 @@ async function down(): Promise<void> {
162
175
  await runCommand("docker", ["compose", "--file", resolve(projectRoot, "compose.yaml"), "down"]);
163
176
  }
164
177
 
178
+ async function associate(args: string[]): Promise<void> {
179
+ const parsed = parseArgs({
180
+ args,
181
+ options: {
182
+ repo: { type: "string", short: "r" },
183
+ pr: { type: "string" },
184
+ session: { type: "string", multiple: true },
185
+ output: { type: "string", short: "o" },
186
+ "list-sessions": { type: "boolean", default: false },
187
+ },
188
+ strict: true,
189
+ });
190
+ const repositoryPath = resolve(parsed.values.repo ?? fail("--repo is required"));
191
+ const [repository, collected] = await Promise.all([
192
+ resolveRepository(repositoryPath),
193
+ collectRepositoryProvenanceWithMetadata(repositoryPath, process.env.HOME ?? homedir()),
194
+ ]);
195
+ const localMessages = collected.messages;
196
+ const sessions = associationSessionSummaries(localMessages, collected.sessions);
197
+ if (parsed.values["list-sessions"]) {
198
+ console.log(JSON.stringify({ repository: repository.url, sessions }, null, 2));
199
+ return;
200
+ }
201
+
202
+ const sourcePr = positiveInteger(parsed.values.pr ?? fail("--pr is required"), "--pr");
203
+ const output = resolve(parsed.values.output ?? fail("--output is required"));
204
+ const selectors = parsed.values.session ?? [];
205
+ const pullRequest = await resolveMergedPullRequest(repository.url, sourcePr);
206
+ const manifest = createProvenanceAssociationManifest({
207
+ repositoryUrl: repository.url,
208
+ pullRequest,
209
+ messages: localMessages,
210
+ sessionSelectors: selectors,
211
+ });
212
+ await writeProvenanceAssociationManifest(output, manifest);
213
+ console.log(
214
+ JSON.stringify(
215
+ {
216
+ output,
217
+ repository: manifest.repository,
218
+ sourcePr: manifest.sourcePr,
219
+ sourceUrl: manifest.sourceUrl,
220
+ sessions: selectors.length,
221
+ messages: manifest.messages.length,
222
+ },
223
+ null,
224
+ 2,
225
+ ),
226
+ );
227
+ }
228
+
165
229
  async function run(args: string[]): Promise<void> {
166
230
  const parsed = parseArgs({
167
231
  args,
@@ -174,6 +238,7 @@ async function run(args: string[]): Promise<void> {
174
238
  model: { type: "string", default: "gpt-5.6-sol" },
175
239
  wait: { type: "boolean", default: false },
176
240
  output: { type: "string", short: "o" },
241
+ association: { type: "string", multiple: true },
177
242
  },
178
243
  strict: true,
179
244
  });
@@ -184,21 +249,25 @@ async function run(args: string[]): Promise<void> {
184
249
  hard: nonnegativeInteger(parsed.values["hard-count"] ?? "0", "--hard-count"),
185
250
  };
186
251
  const totalCandidates = candidateCounts.easy + candidateCounts.medium + candidateCounts.hard;
187
- if (totalCandidates < 1 || totalCandidates > 100) {
188
- fail("the total candidate count must be between 1 and 100");
252
+ if (totalCandidates < 1 || totalCandidates > MAX_CANDIDATES_PER_RUN) {
253
+ fail(`the total candidate count must be between 1 and ${MAX_CANDIDATES_PER_RUN}`);
189
254
  }
190
255
  const runId = parsed.values["run-id"] ?? defaultRunId();
191
- const [repository, localMessages, selfbenchCommit] = await Promise.all([
256
+ const [repository, collectedMessages, selfbenchCommit] = await Promise.all([
192
257
  resolveRepository(repositoryPath),
193
258
  collectRepositoryProvenance(repositoryPath, process.env.HOME ?? homedir()),
194
259
  resolveSelfBenchCommit(),
195
260
  ]);
196
- const githubMessages = await collectGitHubPullRequestProvenance(repository.url);
197
- const messages = [...localMessages, ...githubMessages];
198
- if (messages.length === 0) {
199
- throw new Error("no sanitized local-session or GitHub pull-request provenance was found");
200
- }
201
- const corpus = Buffer.from(`${messages.map((message) => JSON.stringify(message)).join("\n")}\n`);
261
+ const localMessages = await applyProvenanceAssociationManifests(
262
+ collectedMessages,
263
+ repository.url,
264
+ parsed.values.association ?? [],
265
+ );
266
+ const corpus = Buffer.from(
267
+ localMessages.length > 0
268
+ ? `${localMessages.map((message) => JSON.stringify(message)).join("\n")}\n`
269
+ : "",
270
+ );
202
271
  const provenance = await requestJson(`/v1/provenance?runId=${encodeURIComponent(runId)}`, {
203
272
  method: "POST",
204
273
  body: corpus,
@@ -222,9 +291,8 @@ async function run(args: string[]): Promise<void> {
222
291
  JSON.stringify(
223
292
  {
224
293
  ...response,
225
- provenanceMessages: messages.length,
226
294
  localProvenanceMessages: localMessages.length,
227
- githubPullRequestMessages: githubMessages.length,
295
+ associationManifests: parsed.values.association?.length ?? 0,
228
296
  },
229
297
  null,
230
298
  2,
@@ -377,6 +445,14 @@ function asPolledRunStatus(value: Record<string, unknown>): PolledRunStatus {
377
445
  return { ...value, phase: value.phase };
378
446
  }
379
447
 
448
+ function positiveInteger(value: string, label: string): number {
449
+ const parsed = Number(value);
450
+ if (!Number.isInteger(parsed) || parsed < 1) {
451
+ throw new Error(`${label} must be a positive integer`);
452
+ }
453
+ return parsed;
454
+ }
455
+
380
456
  function nonnegativeInteger(value: string, label: string): number {
381
457
  const parsed = Number(value);
382
458
  if (!Number.isInteger(parsed) || parsed < 0) {
@@ -401,8 +477,12 @@ Usage:
401
477
  self-bench up [--backend docker|modal|vercel] [--harbor-environment docker|modal]
402
478
  [--modal-config PATH] [--vercel-profile NAME]
403
479
  self-bench down
480
+ self-bench associate --repo PATH --list-sessions
481
+ self-bench associate --repo PATH --pr NUMBER --session TYPE:SESSION_ID [...]
482
+ --output ASSOCIATION.json
404
483
  self-bench run --repo PATH [--easy-count N] [--medium-count N] [--hard-count N]
405
- [--model MODEL] [--run-id ID] [--wait] [--output OUTPUT.tar.gz]
484
+ [--model MODEL] [--association ASSOCIATION.json ...]
485
+ [--run-id ID] [--wait] [--output OUTPUT.tar.gz]
406
486
  self-bench status RUN_ID
407
487
  self-bench cancel RUN_ID
408
488
  self-bench download RUN_ID OUTPUT.tar.gz
@@ -413,6 +493,10 @@ requires --harbor-environment because Harbor does not support Vercel. Modal gene
413
493
  ~/.modal.toml unless --modal-config overrides it. Run self-bench setup vercel once to create or select a
414
494
  project, publish the pinned runtime image, verify access, and save an owner-only local profile.
415
495
 
496
+ The associate command runs locally. --list-sessions prints selectors, counts, local paths, and modification
497
+ times, but never request text. Association writes a create-only, text-free manifest; it never uploads or
498
+ starts a run. Pass the manifest to run with --association (repeatable). No LLM participates in association.
499
+
416
500
  The tier counts are candidate authoring budgets, not accepted-task targets. Rejected candidates are not
417
501
  replaced, and the export contains only accepted tasks. The run command performs only repository metadata
418
502
  and sanitized provenance upload locally; discovery, authoring, validation, review, and audit run remotely.
package/src/config.ts CHANGED
@@ -6,7 +6,7 @@ import {
6
6
  type HarborEnvironment,
7
7
  matchingHarborEnvironment,
8
8
  } from "./providers.js";
9
- import { parseSandboxTimeoutCapText, STANDARD_VERCEL_TIMEOUT_CAP_MS } from "./sandbox-timeout.js";
9
+ import { parseSandboxTimeoutCapText, STANDARD_VERCEL_TIMEOUT_CAP_MS } from "./sandbox/timeout.js";
10
10
 
11
11
  const emptyStringAsUndefined = (value: unknown): unknown =>
12
12
  typeof value === "string" && value.trim() === "" ? undefined : value;
package/src/contracts.ts CHANGED
@@ -22,17 +22,19 @@ export type ArtifactRef = z.infer<typeof artifactRefSchema>;
22
22
  export const difficultySchema = z.enum(["easy", "medium", "hard"]);
23
23
  export type Difficulty = z.infer<typeof difficultySchema>;
24
24
 
25
+ export const MAX_CANDIDATES_PER_RUN = 10_000;
26
+
25
27
  const candidateCountsSchema = z
26
28
  .object({
27
- easy: z.number().int().min(0).max(100),
28
- medium: z.number().int().min(0).max(100),
29
- hard: z.number().int().min(0).max(100),
29
+ easy: z.number().int().min(0).max(MAX_CANDIDATES_PER_RUN),
30
+ medium: z.number().int().min(0).max(MAX_CANDIDATES_PER_RUN),
31
+ hard: z.number().int().min(0).max(MAX_CANDIDATES_PER_RUN),
30
32
  })
31
33
  .refine(({ easy, medium, hard }) => easy + medium + hard >= 1, {
32
34
  message: "at least one candidate must be requested",
33
35
  })
34
- .refine(({ easy, medium, hard }) => easy + medium + hard <= 100, {
35
- message: "at most 100 candidates may be requested",
36
+ .refine(({ easy, medium, hard }) => easy + medium + hard <= MAX_CANDIDATES_PER_RUN, {
37
+ message: `at most ${MAX_CANDIDATES_PER_RUN} candidates may be requested`,
36
38
  });
37
39
 
38
40
  const runVersionSchema = z
@@ -1,4 +1,4 @@
1
- import { writeFileSync } from "node:fs";
1
+ import { readFileSync, writeFileSync } from "node:fs";
2
2
  import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
3
3
  import { Type } from "@sinclair/typebox";
4
4
 
@@ -22,7 +22,6 @@ export default function discoveryExtension(pi: ExtensionAPI): void {
22
22
  sourceUrl: Type.String({ minLength: 1 }),
23
23
  baseCommit: Type.String({ pattern: "^[0-9a-fA-F]{40}$" }),
24
24
  completedCommit: Type.String({ pattern: "^[0-9a-fA-F]{40}$" }),
25
- request: Type.String({ minLength: 1 }),
26
25
  provenance: Type.Object({
27
26
  sourceType: Type.Union([
28
27
  Type.Literal("pi"),
@@ -51,10 +50,20 @@ export default function discoveryExtension(pi: ExtensionAPI): void {
51
50
  if (!candidates) {
52
51
  throw new Error("submit_discovery received no candidates");
53
52
  }
54
- writeFileSync(output, `${JSON.stringify({ candidates }, null, 2)}\n`, { flag: "wx" });
53
+ const exclusionsPath = process.env.SELFBENCH_DISCOVERY_EXCLUSIONS;
54
+ if (!exclusionsPath) {
55
+ throw new Error("SELFBENCH_DISCOVERY_EXCLUSIONS is not configured");
56
+ }
57
+ const excluded = new Set<number>(JSON.parse(readFileSync(exclusionsPath, "utf8")));
58
+ const accepted = candidates.filter(
59
+ ({ sourcePr }) => sourcePr !== undefined && !excluded.has(sourcePr),
60
+ );
61
+ writeFileSync(output, `${JSON.stringify({ candidates: accepted }, null, 2)}\n`, {
62
+ flag: "wx",
63
+ });
55
64
  return {
56
- content: [{ type: "text", text: `Submitted ${candidates.length} candidates.` }],
57
- details: { candidates },
65
+ content: [{ type: "text", text: `Submitted ${accepted.length} new candidates.` }],
66
+ details: { candidates: accepted },
58
67
  };
59
68
  },
60
69
  });
@@ -6,7 +6,7 @@ import { runCommand } from "./process.js";
6
6
  import { patchPaths } from "./repair.js";
7
7
 
8
8
  const HARBOR_SCHEMA_VERSION = "1.4";
9
- const COMPILER_REVISION = 23;
9
+ const COMPILER_REVISION = 24;
10
10
 
11
11
  export interface AuthoredTaskFiles {
12
12
  readonly definition: TaskDefinition;
@@ -70,6 +70,10 @@ export async function compileHarborTask(
70
70
  await Promise.all([
71
71
  cp(snapshot, join(environment, "repo.tar.gz")),
72
72
  cp(snapshot, join(tests, "repo.tar.gz")),
73
+ writeFile(
74
+ join(outputDirectory, "definition.json"),
75
+ `${JSON.stringify(task.definition, null, 2)}\n`,
76
+ ),
73
77
  writeFile(join(outputDirectory, "instruction.md"), `${task.definition.prompt.trim()}\n`),
74
78
  writeFile(join(solution, "gold.patch"), task.goldPatch),
75
79
  writeFile(join(solution, "solve.sh"), solutionScript()),
@@ -127,6 +131,7 @@ export async function refreshHarborTask(
127
131
  const preinstallGoldDependencies = dependencySetupPatch.length > 0;
128
132
  await writeFile(join(outputDirectory, "tests/test.sh"), testScript(definition, testPatch));
129
133
  await Promise.all([
134
+ writeFile(join(outputDirectory, "definition.json"), `${JSON.stringify(definition, null, 2)}\n`),
130
135
  writeFile(join(outputDirectory, "task.toml"), taskToml(definition)),
131
136
  writeFile(join(outputDirectory, "environment/Dockerfile"), agentDockerfile(definition)),
132
137
  writeFile(
@@ -0,0 +1,15 @@
1
+ import { existsSync } from "node:fs";
2
+ import { dirname, join, parse } from "node:path";
3
+ import { fileURLToPath } from "node:url";
4
+
5
+ export function projectRoot(moduleUrl: string): string {
6
+ let directory = dirname(fileURLToPath(moduleUrl));
7
+ const root = parse(directory).root;
8
+ while (directory !== root) {
9
+ if (existsSync(join(directory, "package.json"))) {
10
+ return directory;
11
+ }
12
+ directory = dirname(directory);
13
+ }
14
+ throw new Error(`could not find package.json above ${fileURLToPath(moduleUrl)}`);
15
+ }
@@ -0,0 +1,311 @@
1
+ import { readFile, writeFile } from "node:fs/promises";
2
+ import { z } from "zod";
3
+ import { assertPullRequestBelongsToRepository, githubRepository } from "./github.js";
4
+ import { sha256 } from "./hash.js";
5
+ import { runCommand } from "./process.js";
6
+ import type {
7
+ LocalSessionMetadata,
8
+ ProvenanceMessage,
9
+ SessionProvenanceFormat,
10
+ } from "./provenance.js";
11
+
12
+ const localSourceTypeSchema = z.enum(["pi", "claude-code", "codex"]);
13
+ type LocalSourceType = z.infer<typeof localSourceTypeSchema>;
14
+
15
+ const associationMessageSchema = z
16
+ .object({
17
+ sourceType: localSourceTypeSchema,
18
+ sessionId: z.string().min(1),
19
+ messageIndex: z.number().int().nonnegative(),
20
+ contentSha256: z.string().regex(/^[0-9a-f]{64}$/),
21
+ })
22
+ .strict();
23
+
24
+ export const provenanceAssociationManifestSchema = z
25
+ .object({
26
+ schemaVersion: z.literal(1),
27
+ repository: z.string().regex(/^[^/]+\/[^/]+$/),
28
+ sourcePr: z.number().int().positive(),
29
+ sourceUrl: z.string().url(),
30
+ messages: z.array(associationMessageSchema).min(1),
31
+ })
32
+ .strict();
33
+
34
+ export type ProvenanceAssociationManifest = z.infer<typeof provenanceAssociationManifestSchema>;
35
+
36
+ export interface AssociationSessionSummary {
37
+ readonly selector: string;
38
+ readonly sourceType: LocalSourceType;
39
+ readonly sessionId: string;
40
+ readonly messageCount: number;
41
+ readonly modifiedAt?: string;
42
+ readonly paths?: readonly string[];
43
+ }
44
+
45
+ export interface MergedPullRequest {
46
+ readonly sourcePr: number;
47
+ readonly sourceUrl: string;
48
+ }
49
+
50
+ export function associationSessionSummaries(
51
+ messages: readonly ProvenanceMessage[],
52
+ metadata: readonly LocalSessionMetadata[] = [],
53
+ ): AssociationSessionSummary[] {
54
+ const metadataBySelector = new Map<string, { modifiedAt: string; paths: Set<string> }>();
55
+ for (const session of metadata) {
56
+ if (!isLocalSourceType(session.sourceType)) {
57
+ continue;
58
+ }
59
+ const selector = sessionSelector(session.sourceType, session.sessionId);
60
+ const existing = metadataBySelector.get(selector);
61
+ metadataBySelector.set(selector, {
62
+ modifiedAt:
63
+ !existing || session.modifiedAt > existing.modifiedAt
64
+ ? session.modifiedAt
65
+ : existing.modifiedAt,
66
+ paths: new Set([...(existing?.paths ?? []), session.path]),
67
+ });
68
+ }
69
+
70
+ const counts = new Map<string, AssociationSessionSummary>();
71
+ for (const message of messages) {
72
+ if (!isLocalSourceType(message.sourceType)) {
73
+ continue;
74
+ }
75
+ const selector = sessionSelector(message.sourceType, message.sessionId);
76
+ const existing = counts.get(selector);
77
+ const sessionMetadata = metadataBySelector.get(selector);
78
+ counts.set(selector, {
79
+ selector,
80
+ sourceType: message.sourceType,
81
+ sessionId: message.sessionId,
82
+ messageCount: (existing?.messageCount ?? 0) + 1,
83
+ ...(sessionMetadata
84
+ ? {
85
+ modifiedAt: sessionMetadata.modifiedAt,
86
+ paths: [...sessionMetadata.paths].sort(),
87
+ }
88
+ : {}),
89
+ });
90
+ }
91
+ return [...counts.values()].sort(
92
+ (left, right) =>
93
+ (right.modifiedAt ?? "").localeCompare(left.modifiedAt ?? "") ||
94
+ left.selector.localeCompare(right.selector),
95
+ );
96
+ }
97
+
98
+ export function createProvenanceAssociationManifest(input: {
99
+ readonly repositoryUrl: string;
100
+ readonly pullRequest: MergedPullRequest;
101
+ readonly messages: readonly ProvenanceMessage[];
102
+ readonly sessionSelectors: readonly string[];
103
+ }): ProvenanceAssociationManifest {
104
+ const repository = githubRepository(input.repositoryUrl);
105
+ assertPullRequestBelongsToRepository(
106
+ input.repositoryUrl,
107
+ input.pullRequest.sourceUrl,
108
+ input.pullRequest.sourcePr,
109
+ );
110
+ if (input.sessionSelectors.length === 0) {
111
+ throw new Error("at least one --session is required to create an association manifest");
112
+ }
113
+
114
+ const selected = new Set(input.sessionSelectors.map(parseSessionSelector));
115
+ if (selected.size !== input.sessionSelectors.length) {
116
+ throw new Error("each --session selector must be unique");
117
+ }
118
+ const available = new Set(
119
+ associationSessionSummaries(input.messages).map(({ selector }) => selector),
120
+ );
121
+ for (const selector of selected) {
122
+ if (!available.has(selector)) {
123
+ throw new Error(`local session ${selector} was not found for this repository`);
124
+ }
125
+ }
126
+
127
+ const messageKeys = new Set<string>();
128
+ const messages = input.messages
129
+ .filter(
130
+ (message): message is ProvenanceMessage & { sourceType: LocalSourceType } =>
131
+ isLocalSourceType(message.sourceType) &&
132
+ selected.has(sessionSelector(message.sourceType, message.sessionId)),
133
+ )
134
+ .map((message) => {
135
+ const key = messageKey(message.sourceType, message.sessionId, message.messageIndex);
136
+ if (messageKeys.has(key)) {
137
+ throw new Error(`local provenance contains duplicate message identity ${key}`);
138
+ }
139
+ messageKeys.add(key);
140
+ return {
141
+ sourceType: message.sourceType,
142
+ sessionId: message.sessionId,
143
+ messageIndex: message.messageIndex,
144
+ contentSha256: sha256(message.content),
145
+ };
146
+ })
147
+ .sort(compareAssociationMessages);
148
+
149
+ return provenanceAssociationManifestSchema.parse({
150
+ schemaVersion: 1,
151
+ repository,
152
+ sourcePr: input.pullRequest.sourcePr,
153
+ sourceUrl: input.pullRequest.sourceUrl,
154
+ messages,
155
+ });
156
+ }
157
+
158
+ export async function writeProvenanceAssociationManifest(
159
+ path: string,
160
+ manifest: ProvenanceAssociationManifest,
161
+ ): Promise<void> {
162
+ await writeFile(path, `${JSON.stringify(manifest, null, 2)}\n`, { flag: "wx", mode: 0o600 });
163
+ }
164
+
165
+ export async function applyProvenanceAssociationManifests(
166
+ messages: readonly ProvenanceMessage[],
167
+ repositoryUrl: string,
168
+ paths: readonly string[],
169
+ ): Promise<ProvenanceMessage[]> {
170
+ if (paths.length === 0) {
171
+ return [...messages];
172
+ }
173
+ const repository = githubRepository(repositoryUrl);
174
+ const manifests = await Promise.all(
175
+ paths.map(async (path) => {
176
+ const value = JSON.parse(await readFile(path, "utf8"));
177
+ return { path, manifest: provenanceAssociationManifestSchema.parse(value) };
178
+ }),
179
+ );
180
+ const messageByKey = new Map<string, ProvenanceMessage>();
181
+ for (const message of messages) {
182
+ const key = messageKey(message.sourceType, message.sessionId, message.messageIndex);
183
+ if (messageByKey.has(key)) {
184
+ throw new Error(`local provenance contains duplicate message identity ${key}`);
185
+ }
186
+ messageByKey.set(key, message);
187
+ }
188
+
189
+ const associations = new Map<string, MergedPullRequest>();
190
+ for (const { path, manifest } of manifests) {
191
+ if (manifest.repository.toLowerCase() !== repository) {
192
+ throw new Error(
193
+ `association manifest ${path} belongs to ${manifest.repository}, not ${repository}`,
194
+ );
195
+ }
196
+ assertPullRequestBelongsToRepository(repositoryUrl, manifest.sourceUrl, manifest.sourcePr);
197
+ for (const selector of new Set(
198
+ manifest.messages.map((message) => sessionSelector(message.sourceType, message.sessionId)),
199
+ )) {
200
+ const expected = manifest.messages
201
+ .filter((message) => sessionSelector(message.sourceType, message.sessionId) === selector)
202
+ .sort(compareAssociationMessages);
203
+ const current = messages
204
+ .filter(
205
+ (message): message is ProvenanceMessage & { sourceType: LocalSourceType } =>
206
+ isLocalSourceType(message.sourceType) &&
207
+ sessionSelector(message.sourceType, message.sessionId) === selector,
208
+ )
209
+ .map((message) => ({
210
+ sourceType: message.sourceType,
211
+ sessionId: message.sessionId,
212
+ messageIndex: message.messageIndex,
213
+ contentSha256: sha256(message.content),
214
+ }))
215
+ .sort(compareAssociationMessages);
216
+ if (JSON.stringify(current) !== JSON.stringify(expected)) {
217
+ throw new Error(`association manifest ${path} does not match local session ${selector}`);
218
+ }
219
+ }
220
+ for (const reference of manifest.messages) {
221
+ const key = messageKey(reference.sourceType, reference.sessionId, reference.messageIndex);
222
+ const message = messageByKey.get(key);
223
+ if (!message || sha256(message.content) !== reference.contentSha256) {
224
+ throw new Error(`association manifest ${path} does not match local message ${key}`);
225
+ }
226
+ if (associations.has(key)) {
227
+ throw new Error(`local message ${key} is associated more than once`);
228
+ }
229
+ associations.set(key, {
230
+ sourcePr: manifest.sourcePr,
231
+ sourceUrl: manifest.sourceUrl,
232
+ });
233
+ }
234
+ }
235
+
236
+ return messages.map((message) => {
237
+ const association = associations.get(
238
+ messageKey(message.sourceType, message.sessionId, message.messageIndex),
239
+ );
240
+ return association ? { ...message, ...association } : message;
241
+ });
242
+ }
243
+
244
+ export async function resolveMergedPullRequest(
245
+ repositoryUrl: string,
246
+ sourcePr: number,
247
+ ): Promise<MergedPullRequest> {
248
+ const repository = githubRepository(repositoryUrl);
249
+ const result = await runCommand("gh", [
250
+ "pr",
251
+ "view",
252
+ String(sourcePr),
253
+ "--repo",
254
+ repository,
255
+ "--json",
256
+ "number,url,state,mergedAt",
257
+ ]);
258
+ const parsed = z
259
+ .object({
260
+ number: z.number().int().positive(),
261
+ url: z.string().url(),
262
+ state: z.string(),
263
+ mergedAt: z.string().nullable(),
264
+ })
265
+ .parse(JSON.parse(result.stdout));
266
+ if (parsed.number !== sourcePr) {
267
+ throw new Error(`GitHub returned pull request ${parsed.number}; expected ${sourcePr}`);
268
+ }
269
+ assertPullRequestBelongsToRepository(repositoryUrl, parsed.url, sourcePr);
270
+ if (parsed.state !== "MERGED" || !parsed.mergedAt) {
271
+ throw new Error(`pull request ${repository}#${sourcePr} is not merged`);
272
+ }
273
+ return { sourcePr, sourceUrl: parsed.url };
274
+ }
275
+
276
+ function parseSessionSelector(value: string): string {
277
+ const separator = value.indexOf(":");
278
+ if (separator < 1 || separator === value.length - 1) {
279
+ throw new Error(`invalid session selector ${JSON.stringify(value)}; expected TYPE:SESSION_ID`);
280
+ }
281
+ const sourceType = value.slice(0, separator);
282
+ localSourceTypeSchema.parse(sourceType);
283
+ return sessionSelector(sourceType as LocalSourceType, value.slice(separator + 1));
284
+ }
285
+
286
+ function sessionSelector(sourceType: LocalSourceType, sessionId: string): string {
287
+ return `${sourceType}:${sessionId}`;
288
+ }
289
+
290
+ function messageKey(
291
+ sourceType: SessionProvenanceFormat | "github-pull-request",
292
+ sessionId: string,
293
+ messageIndex: number,
294
+ ): string {
295
+ return `${sourceType}:${sessionId}:${messageIndex}`;
296
+ }
297
+
298
+ function compareAssociationMessages(
299
+ left: z.infer<typeof associationMessageSchema>,
300
+ right: z.infer<typeof associationMessageSchema>,
301
+ ): number {
302
+ return (
303
+ left.sourceType.localeCompare(right.sourceType) ||
304
+ left.sessionId.localeCompare(right.sessionId) ||
305
+ left.messageIndex - right.messageIndex
306
+ );
307
+ }
308
+
309
+ function isLocalSourceType(value: string): value is LocalSourceType {
310
+ return value === "pi" || value === "claude-code" || value === "codex";
311
+ }