toolroll 0.9.81 → 0.9.83

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/CHANGELOG.md +35 -0
  2. package/dist/cli.js +1 -0
  3. package/dist/commit-extract.d.ts +18 -0
  4. package/dist/commit-extract.js +39 -1
  5. package/dist/converse.d.ts +12 -0
  6. package/dist/converse.js +18 -2
  7. package/dist/deploy-gate.js +8 -0
  8. package/dist/desktop-update.d.ts +1 -1
  9. package/dist/eval-gate.d.ts +51 -0
  10. package/dist/eval-gate.js +199 -0
  11. package/dist/eval-hooks.d.ts +39 -0
  12. package/dist/eval-hooks.js +477 -0
  13. package/dist/eval-oracle-main.d.ts +1 -0
  14. package/dist/eval-oracle-main.js +31 -0
  15. package/dist/eval-oracle.d.ts +60 -0
  16. package/dist/eval-oracle.js +217 -0
  17. package/dist/eval-world.d.ts +17 -0
  18. package/dist/eval-world.js +97 -0
  19. package/dist/evals.d.ts +299 -0
  20. package/dist/evals.js +964 -0
  21. package/dist/exec.js +15 -1
  22. package/dist/first-tasks-child.d.ts +1 -0
  23. package/dist/first-tasks-child.js +11 -0
  24. package/dist/flow-draft.js +4 -0
  25. package/dist/flow-sort.js +4 -0
  26. package/dist/folder-look.d.ts +107 -0
  27. package/dist/folder-look.js +302 -0
  28. package/dist/github-jobs.d.ts +25 -0
  29. package/dist/github-jobs.js +22 -0
  30. package/dist/invoke.d.ts +1 -1
  31. package/dist/invoke.js +14 -0
  32. package/dist/lead.js +4 -1
  33. package/dist/learning-child.d.ts +1 -0
  34. package/dist/learning-child.js +34 -0
  35. package/dist/learning-look.d.ts +25 -0
  36. package/dist/learning-look.js +72 -0
  37. package/dist/memory-pass.js +4 -0
  38. package/dist/nightly-check.d.ts +4 -1
  39. package/dist/nightly-check.js +5 -2
  40. package/dist/nightly-github.d.ts +316 -0
  41. package/dist/nightly-github.js +1022 -0
  42. package/dist/operate.d.ts +123 -1
  43. package/dist/operate.js +344 -29
  44. package/dist/project-identity.d.ts +37 -0
  45. package/dist/project-identity.js +106 -0
  46. package/dist/project-knowledge.d.ts +4 -4
  47. package/dist/project-tools.d.ts +4 -1
  48. package/dist/project-tools.js +7 -2
  49. package/dist/project.d.ts +4 -1
  50. package/dist/project.js +11 -4
  51. package/dist/provider.d.ts +3 -0
  52. package/dist/provider.js +9 -2
  53. package/dist/release-adapters.d.ts +21 -21
  54. package/dist/release-adapters.js +52 -19
  55. package/dist/release-auto.js +26 -0
  56. package/dist/release-card.js +3 -0
  57. package/dist/release-cli.js +59 -3
  58. package/dist/release-view.d.ts +3 -0
  59. package/dist/release-view.js +2 -0
  60. package/dist/release.d.ts +16 -0
  61. package/dist/release.js +16 -0
  62. package/dist/reported-usage.d.ts +69 -0
  63. package/dist/reported-usage.js +97 -0
  64. package/dist/repos.d.ts +7 -0
  65. package/dist/repos.js +57 -0
  66. package/dist/serve.js +35 -8
  67. package/dist/server/chat.js +4 -0
  68. package/dist/server/flows.js +9 -3
  69. package/dist/server/guards.d.ts +4 -0
  70. package/dist/server/guards.js +12 -3
  71. package/dist/server/http.d.ts +11 -0
  72. package/dist/server/http.js +3 -0
  73. package/dist/server/pages.js +62 -88
  74. package/dist/server/render-pages.d.ts +2 -16
  75. package/dist/server/render-pages.js +1 -55
  76. package/dist/server/render-tasks.js +1 -0
  77. package/dist/server/runtime.d.ts +4 -0
  78. package/dist/server/runtime.js +1 -0
  79. package/dist/server/settings.js +14 -3
  80. package/dist/server/tasks.js +15 -4
  81. package/dist/store.d.ts +109 -1
  82. package/dist/store.js +394 -4
  83. package/dist/subagents.js +4 -0
  84. package/dist/subscription-chat.d.ts +2 -0
  85. package/dist/subscription-chat.js +52 -13
  86. package/dist/task-sizing.js +4 -0
  87. package/dist/teardown-git.d.ts +9 -2
  88. package/dist/teardown-git.js +56 -33
  89. package/dist/up-processes.d.ts +13 -0
  90. package/dist/up-processes.js +6 -2
  91. package/evals/cases/catalog/case.json +25 -0
  92. package/evals/cases/catalog/fixture/README.md +3 -0
  93. package/evals/cases/catalog/fixture/package.json +8 -0
  94. package/evals/cases/catalog/fixture/test/catalog.test.mjs +4 -0
  95. package/evals/cases/catalog/oracle.mjs +23 -0
  96. package/evals/cases/catalog/probe.mjs +10 -0
  97. package/evals/cases/catalog/source.md +9 -0
  98. package/evals/cases/csv-quoting/case.json +23 -0
  99. package/evals/cases/csv-quoting/fixture/README.md +3 -0
  100. package/evals/cases/csv-quoting/fixture/csv.js +1 -0
  101. package/evals/cases/csv-quoting/fixture/package.json +8 -0
  102. package/evals/cases/csv-quoting/fixture/test/csv.test.mjs +5 -0
  103. package/evals/cases/csv-quoting/oracle.mjs +14 -0
  104. package/evals/cases/csv-quoting/probe.mjs +9 -0
  105. package/evals/cases/csv-quoting/source.md +9 -0
  106. package/evals/cases/dedupe/case.json +23 -0
  107. package/evals/cases/dedupe/fixture/README.md +3 -0
  108. package/evals/cases/dedupe/fixture/dedupe.js +1 -0
  109. package/evals/cases/dedupe/fixture/package.json +8 -0
  110. package/evals/cases/dedupe/fixture/test/dedupe.test.mjs +5 -0
  111. package/evals/cases/dedupe/oracle.mjs +16 -0
  112. package/evals/cases/dedupe/probe.mjs +13 -0
  113. package/evals/cases/dedupe/source.md +9 -0
  114. package/evals/cases/endpoint/case.json +25 -0
  115. package/evals/cases/endpoint/fixture/README.md +3 -0
  116. package/evals/cases/endpoint/fixture/package.json +8 -0
  117. package/evals/cases/endpoint/fixture/test/endpoint.test.mjs +4 -0
  118. package/evals/cases/endpoint/oracle.mjs +27 -0
  119. package/evals/cases/endpoint/probe.mjs +10 -0
  120. package/evals/cases/endpoint/source.md +11 -0
  121. package/evals/cases/math-divide/case.json +23 -0
  122. package/evals/cases/math-divide/fixture/README.md +3 -0
  123. package/evals/cases/math-divide/fixture/package.json +8 -0
  124. package/evals/cases/math-divide/fixture/src/math.js +1 -0
  125. package/evals/cases/math-divide/fixture/test/math.test.js +5 -0
  126. package/evals/cases/math-divide/oracle.mjs +13 -0
  127. package/evals/cases/math-divide/probe.mjs +9 -0
  128. package/evals/cases/math-divide/source.md +9 -0
  129. package/evals/cases/math-multiply/case.json +23 -0
  130. package/evals/cases/math-multiply/fixture/README.md +3 -0
  131. package/evals/cases/math-multiply/fixture/package.json +8 -0
  132. package/evals/cases/math-multiply/fixture/src/math.js +2 -0
  133. package/evals/cases/math-multiply/fixture/test/math.test.js +5 -0
  134. package/evals/cases/math-multiply/oracle.mjs +12 -0
  135. package/evals/cases/math-multiply/probe.mjs +9 -0
  136. package/evals/cases/math-multiply/source.md +9 -0
  137. package/evals/cases/math-subtract/case.json +23 -0
  138. package/evals/cases/math-subtract/fixture/README.md +3 -0
  139. package/evals/cases/math-subtract/fixture/package.json +8 -0
  140. package/evals/cases/math-subtract/fixture/src/math.js +1 -0
  141. package/evals/cases/math-subtract/fixture/test/math.test.js +5 -0
  142. package/evals/cases/math-subtract/oracle.mjs +12 -0
  143. package/evals/cases/math-subtract/probe.mjs +9 -0
  144. package/evals/cases/math-subtract/source.md +9 -0
  145. package/evals/cases/merge-config/case.json +23 -0
  146. package/evals/cases/merge-config/fixture/README.md +3 -0
  147. package/evals/cases/merge-config/fixture/config.js +1 -0
  148. package/evals/cases/merge-config/fixture/package.json +8 -0
  149. package/evals/cases/merge-config/fixture/test/config.test.mjs +5 -0
  150. package/evals/cases/merge-config/oracle.mjs +28 -0
  151. package/evals/cases/merge-config/probe.mjs +15 -0
  152. package/evals/cases/merge-config/source.md +9 -0
  153. package/evals/cases/paginate/case.json +23 -0
  154. package/evals/cases/paginate/fixture/README.md +3 -0
  155. package/evals/cases/paginate/fixture/package.json +8 -0
  156. package/evals/cases/paginate/fixture/paginate.js +1 -0
  157. package/evals/cases/paginate/fixture/test/paginate.test.mjs +5 -0
  158. package/evals/cases/paginate/oracle.mjs +18 -0
  159. package/evals/cases/paginate/probe.mjs +13 -0
  160. package/evals/cases/paginate/source.md +9 -0
  161. package/evals/cases/round2-asks/case.json +24 -0
  162. package/evals/cases/round2-asks/fixture/README.md +3 -0
  163. package/evals/cases/round2-asks/fixture/package.json +8 -0
  164. package/evals/cases/round2-asks/fixture/src/math.js +1 -0
  165. package/evals/cases/round2-asks/fixture/test/math.test.js +5 -0
  166. package/evals/cases/round2-asks/source.md +11 -0
  167. package/evals/manifest.json +15 -0
  168. package/package.json +2 -1
@@ -10,6 +10,7 @@
10
10
  * are still interpreted and executed by lead.ts, where every act remains a
11
11
  * proposal that needs a human confirmation card.
12
12
  */
13
+ import { takeEvalDispatch } from "./eval-gate.js";
13
14
  import { Buffer } from "node:buffer";
14
15
  import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
15
16
  import { tmpdir } from "node:os";
@@ -22,6 +23,7 @@ import { ALL_CREDENTIAL_ENV } from "./provider.js";
22
23
  import { claudeLimitsOf, noteLimits } from "./provider-limits.js";
23
24
  import { claudeBillingFrom } from "./spend.js";
24
25
  import { primaryResultEvent, topLevelEvent } from "./stream-scope.js";
26
+ import { claudeResultReport, codexUsageReport, reportedModelOf } from "./reported-usage.js";
25
27
  const RESPONSE_CAP_BYTES = 65_536;
26
28
  function outputSchema(tools) {
27
29
  return {
@@ -108,12 +110,29 @@ export function parseSubscriptionLeadAnswer(text, usage = {}) {
108
110
  }
109
111
  const tokensIn = typeof usage.tokensIn === "number" && tokenCount(usage.tokensIn) ? usage.tokensIn : 0;
110
112
  const tokensOut = typeof usage.tokensOut === "number" && tokenCount(usage.tokensOut) ? usage.tokensOut : 0;
111
- return { ok: true, answer: { text: body["text"], calls, tokensIn, tokensOut, reportedCostMicrousd: null } };
113
+ return { ok: true, answer: { text: body["text"], calls, tokensIn, tokensOut, reportedCostMicrousd: null, ...reportFields(usage.report ?? null) } };
114
+ }
115
+ /** The harness's own report on the answer (v140): every field null when it gave none, cache counts only when it
116
+ * reported them. Never `reportedCostMicrousd`: a subscription's reported cost is usage, not API dollars. */
117
+ function reportFields(report) {
118
+ return {
119
+ modelReported: report?.modelReported ?? null,
120
+ providerTurns: report?.providerTurns ?? null,
121
+ reportedCostUsd: report?.reportedCostUsd ?? null,
122
+ ...(report?.cacheReadTokens == null ? {} : { cacheReadTokens: report.cacheReadTokens }),
123
+ ...(report?.cacheWriteTokens == null ? {} : { cacheWriteTokens: report.cacheWriteTokens }),
124
+ };
125
+ }
126
+ /** Input tokens are everything the model read, cache included, when the harness reported every part (usageOf);
127
+ * otherwise its plain input count as before. */
128
+ function tokensInOf(report, plain) {
129
+ return report.tokensIn !== null && tokenCount(report.tokensIn) ? report.tokensIn : plain;
112
130
  }
113
131
  function codexOutput(stdout) {
114
132
  let text = null;
115
133
  let tokensIn = 0;
116
134
  let tokensOut = 0;
135
+ let report = null;
117
136
  let completed = 0;
118
137
  let invalid = false;
119
138
  for (const line of stdout.split("\n")) {
@@ -148,27 +167,31 @@ function codexOutput(stdout) {
148
167
  tokensIn = usage["input_tokens"];
149
168
  if (typeof usage?.["output_tokens"] === "number" && tokenCount(usage["output_tokens"]))
150
169
  tokensOut = usage["output_tokens"];
170
+ report = codexUsageReport(usage);
151
171
  }
152
172
  }
153
173
  // A partial answer is not authority to run host tools. Process exit zero
154
174
  // alone is insufficient; the harness must finish this one turn successfully.
155
- return { text: !invalid && completed === 1 ? text : null, tokensIn, tokensOut };
175
+ return { text: !invalid && completed === 1 ? text : null, tokensIn: report === null ? tokensIn : tokensInOf(report, tokensIn), tokensOut, report };
156
176
  }
157
- function claudeOutput(stdout) {
177
+ /** The closing `result` (the buffered run's whole output, or the streamed run's last primary result line), with the
178
+ * session's init model when the stream carried one. */
179
+ function claudeOutput(stdout, init) {
158
180
  const parsed = strictJsonParse(Buffer.from(stdout, "utf8"), RESPONSE_CAP_BYTES, 12);
159
181
  if (!parsed.ok || typeof parsed.value !== "object" || parsed.value === null || Array.isArray(parsed.value))
160
- return { text: null, tokensIn: 0, tokensOut: 0 };
182
+ return { text: null, tokensIn: 0, tokensOut: 0, report: null };
161
183
  const body = parsed.value;
162
184
  if (body["type"] !== "result" || body["subtype"] !== "success" || body["is_error"] !== false) {
163
- return { text: null, tokensIn: 0, tokensOut: 0 };
185
+ return { text: null, tokensIn: 0, tokensOut: 0, report: null };
164
186
  }
165
187
  const usage = body["usage"];
166
- const tokensIn = typeof usage?.["input_tokens"] === "number" && tokenCount(usage["input_tokens"]) ? usage["input_tokens"] : 0;
188
+ const report = claudeResultReport(body, init);
189
+ const tokensIn = tokensInOf(report, typeof usage?.["input_tokens"] === "number" && tokenCount(usage["input_tokens"]) ? usage["input_tokens"] : 0);
167
190
  const tokensOut = typeof usage?.["output_tokens"] === "number" && tokenCount(usage["output_tokens"]) ? usage["output_tokens"] : 0;
168
191
  const structured = body["structured_output"];
169
192
  if (typeof structured === "object" && structured !== null)
170
- return { text: JSON.stringify(structured), tokensIn, tokensOut };
171
- return { text: typeof body["result"] === "string" ? body["result"] : null, tokensIn, tokensOut };
193
+ return { text: JSON.stringify(structured), tokensIn, tokensOut, report };
194
+ return { text: typeof body["result"] === "string" ? body["result"] : null, tokensIn, tokensOut, report };
172
195
  }
173
196
  /** The streamed run's closing `result` event: the same body the buffered
174
197
  * `--output-format json` run prints. */
@@ -190,6 +213,11 @@ function lastResultLine(stdout) {
190
213
  }
191
214
  /** Production runner; injectable so tests never consume a subscription turn. */
192
215
  export async function performSubscriptionLeadRequest(request, runner = run) {
216
+ // An eval world (eval-gate.ts): each request takes one of the attempt's dispatch slots; past the limit nothing is
217
+ // sent ("status-": nothing billed).
218
+ const evalSlot = takeEvalDispatch(`chat:${request.provider}`);
219
+ if (evalSlot.active && !evalSlot.ok)
220
+ return { ok: false, problem: "status-eval-dispatch-limit" };
193
221
  const dir = mkdtempSync(join(tmpdir(), "standing-orders-mate-"));
194
222
  try {
195
223
  const prompt = composeSubscriptionLeadPrompt(request);
@@ -237,12 +265,16 @@ export async function performSubscriptionLeadRequest(request, runner = run) {
237
265
  if (result.notFound)
238
266
  return { ok: false, problem: "not-found" };
239
267
  // v105: a streamed Claude turn says its plan's usage windows, and (once it answered) how this computer's Claude bills
240
- // with no key from us: the chat ran with every key left out.
268
+ // with no key from us: the chat ran with every key left out. v140: its session's init names the exact model.
269
+ let init = { model: null, conflict: false };
270
+ // v141: how this request was billed, as its own events said (streamed Claude only); null when they didn't say.
271
+ let billingReported = null;
241
272
  if (command === "claude" && streaming) {
242
273
  const seen = { keySource: null, model: null, planWindows: false };
274
+ const initModels = new Set();
243
275
  let answered = false;
244
276
  for (const line of result.stdout.split("\n")) {
245
- if (!line.includes('"rate_limit_event"') && !line.includes('"apiKeySource"') && !line.includes('"type":"result"'))
277
+ if (!line.includes('"rate_limit_event"') && !line.includes('"apiKeySource"') && !line.includes('"init"') && !line.includes('"type":"result"'))
246
278
  continue;
247
279
  let event;
248
280
  try {
@@ -259,23 +291,30 @@ export async function performSubscriptionLeadRequest(request, runner = run) {
259
291
  else if (event["type"] === "system" && event["subtype"] === "init" && topLevelEvent(event)) {
260
292
  seen.keySource = typeof event["apiKeySource"] === "string" ? event["apiKeySource"] : null;
261
293
  seen.model = typeof event["model"] === "string" ? event["model"] : null;
294
+ // Every init the session sent names its model; inits that disagree leave it unknown, never chosen.
295
+ const named = reportedModelOf(event["model"]);
296
+ if (named !== null)
297
+ initModels.add(named);
262
298
  }
263
299
  else if (primaryResultEvent(event))
264
300
  answered = true;
265
301
  }
302
+ init = { model: initModels.size === 1 ? [...initModels][0] : null, conflict: initModels.size > 1 };
266
303
  const billing = answered ? claudeBillingFrom(seen) : null;
267
304
  if (billing !== null)
268
305
  noteLimits({ provider: "claude", plan: null, windows: [], partial: true, billing });
306
+ billingReported = billing;
269
307
  }
270
308
  if (result.code !== 0)
271
309
  return { ok: false, problem: `status-${result.code}` };
272
- const output = request.provider === "codex-subscription" ? codexOutput(result.stdout) : claudeOutput(streaming ? lastResultLine(result.stdout) : result.stdout);
310
+ const output = request.provider === "codex-subscription" ? codexOutput(result.stdout) : claudeOutput(streaming ? lastResultLine(result.stdout) : result.stdout, init);
273
311
  if (output.text === null)
274
312
  return { ok: false, problem: "malformed-reply" };
275
313
  if (request.outputSchema !== undefined) {
276
- return { ok: true, answer: { text: output.text, calls: [], tokensIn: output.tokensIn, tokensOut: output.tokensOut, reportedCostMicrousd: null } };
314
+ return { ok: true, answer: { text: output.text, calls: [], tokensIn: output.tokensIn, tokensOut: output.tokensOut, reportedCostMicrousd: null, ...reportFields(output.report), billingReported } };
277
315
  }
278
- return parseSubscriptionLeadAnswer(output.text, output);
316
+ const parsed = parseSubscriptionLeadAnswer(output.text, output);
317
+ return parsed.ok ? { ok: true, answer: { ...parsed.answer, billingReported } } : parsed;
279
318
  }
280
319
  finally {
281
320
  rmSync(dir, { recursive: true, force: true });
@@ -15,6 +15,7 @@
15
15
  * The task text is data to the classifier, never instructions, and all it
16
16
  * can return is one of three sizes, a yes/no, and a short reason.
17
17
  */
18
+ import { inEvalWorld } from "./eval-gate.js";
18
19
  import { randomUUID } from "node:crypto";
19
20
  import { mkdtempSync, rmSync } from "node:fs";
20
21
  import { tmpdir } from "node:os";
@@ -75,6 +76,9 @@ export function claudeSizer(runner = run, model = SIZING_CLAUDE_MODEL, budgetMs
75
76
  return async (input, signal) => {
76
77
  if (signal.aborted)
77
78
  return null;
79
+ // An eval world makes no provider call it can't cap and record (eval-gate.ts): the task goes unsized.
80
+ if (inEvalWorld())
81
+ return null;
78
82
  const owner = `sizing:${randomUUID()}`;
79
83
  const stop = () => { terminateOwnedProcesses(owner); };
80
84
  signal.addEventListener("abort", stop, { once: true });
@@ -38,7 +38,11 @@ export declare function teardownGit(runner?: Runner, binding?: GitBinding, lockT
38
38
  * git at any file, a pipe that never ends among them. Each file is read only as the regular file it is, bounded; one
39
39
  * that can't be read so counts as including.
40
40
  */
41
- export declare function checkoutIncludes(checkout: string, gitFile?: string): boolean;
41
+ /** What is left of a caller's deadline, in ms (viewedGitSync's `deadline`): absent, no deadline. */
42
+ type Limit = () => number;
43
+ export declare function checkoutIncludes(checkout: string, gitFile?: string, limit?: Limit): boolean;
44
+ /** What includeIn answers when Git didn't finish parsing the file in time: still a refusal, named as what it is. */
45
+ export declare const INCLUDE_PARSE_TIMED_OUT = "Git didn't finish reading it in time";
42
46
  export type GitBinding = {
43
47
  repo?: string | null;
44
48
  };
@@ -50,7 +54,7 @@ export declare const WRITE_VIEW_FLAGS: readonly string[];
50
54
  /** Whether the checkout's .lfsconfig (read as git parses it, that file only) sets any git-lfs key that could name a
51
55
  * program (anything but INERT_LFS). A file that
52
56
  * isn't a plain one, or doesn't parse, counts as naming one. */
53
- export declare function lfsExtensionIn(checkout: string, gitFile?: string): boolean;
57
+ export declare function lfsExtensionIn(checkout: string, gitFile?: string, limit?: Limit): boolean;
54
58
  /** Every command runs against a view of the checkout's repository (gitView) whose config and info are copies the
55
59
  * plane made, inspected there, so neither can change between the inspection and the run. `mode` "read" is for reading
56
60
  * a checkout an agent may still be writing (the code page): every command also takes readFlags(), and the view holds
@@ -88,6 +92,7 @@ export declare function checkoutGitSync(cwd: string, args: readonly string[], op
88
92
  */
89
93
  export declare function viewedGitSync(cwd: string, args: readonly string[], options: {
90
94
  timeoutMs?: number;
95
+ deadline?: number;
91
96
  maxBuffer?: number;
92
97
  binding?: GitBinding;
93
98
  }): {
@@ -96,6 +101,7 @@ export declare function viewedGitSync(cwd: string, args: readonly string[], opti
96
101
  };
97
102
  export declare function viewedGitSync(cwd: string, args: readonly string[], options: {
98
103
  timeoutMs?: number;
104
+ deadline?: number;
99
105
  maxBuffer?: number;
100
106
  binding?: GitBinding;
101
107
  buffer: true;
@@ -137,3 +143,4 @@ export declare function lazyFetchSetting(listing: string): string | null;
137
143
  * command and protocol settings refused, a partial clone refused) is the config git then runs with.
138
144
  */
139
145
  export declare function lazyFetchSafeGit(inner?: Runner): Runner;
146
+ export {};
@@ -290,28 +290,19 @@ export function teardownGit(runner = run, binding = {}, lockTimeoutMs, gitFile =
290
290
  }, !removal, lockTimeoutMs, gitFile);
291
291
  }, true, lockTimeoutMs, gitFile);
292
292
  }
293
- /**
294
- * Git on a build's checkout, as the plane runs it outside every fence (staging, committing and inspecting the build's
295
- * work): the same fixed hardening as teardown (no hooks, fsmonitor, ext transport, credential helpers or automatic
296
- * maintenance, no submodule recursion, no signing), refused outright when the checkout's own config includes a file
297
- * or names a command — a filter, an external diff, a textconv, a pager — which a build could have planted. It reads no
298
- * global or system config: only their values (identity, line endings, a standard git-lfs install) are carried over.
299
- * It never enters a submodule or nested repository (whose config it never read).
300
- */
301
- /**
302
- * Whether a checkout's own git config (its repository's config and its worktree's config.worktree) includes another
303
- * file, read without git: every git command follows includes as it starts, so an include of the build's could point
304
- * git at any file, a pipe that never ends among them. Each file is read only as the regular file it is, bounded; one
305
- * that can't be read so counts as including.
306
- */
307
- export function checkoutIncludes(checkout, gitFile = "git") {
293
+ /** A setup call's time limit: its own five seconds, or less when a caller's deadline leaves less (never below 1 ms;
294
+ * a caller checks the deadline itself before calling). */
295
+ const setupTimeout = (limit) => limit === undefined ? 5_000 : Math.max(1, Math.min(5_000, limit()));
296
+ export function checkoutIncludes(checkout, gitFile = "git", limit) {
308
297
  const dirs = gitDirsOf(checkout);
309
298
  if (dirs === null)
310
299
  return true;
311
300
  for (const path of [join(dirs.common, "config"), join(dirs.gitdir, "config.worktree")]) {
312
301
  if (!existsSync(path) && path.endsWith("config.worktree"))
313
302
  continue;
314
- if (includeIn(path, gitFile) !== null)
303
+ if (limit !== undefined && limit() <= 0)
304
+ return true;
305
+ if (includeIn(path, gitFile, limit) !== null)
315
306
  return true;
316
307
  }
317
308
  return false;
@@ -358,13 +349,23 @@ function gitDirsOf(checkout) {
358
349
  const named = regularFile(join(gitdir, "commondir"))?.trim();
359
350
  return { gitdir, common: named === undefined || named === "" ? gitdir : resolve(gitdir, named) };
360
351
  }
352
+ /** What includeIn answers when Git didn't finish parsing the file in time: still a refusal, named as what it is. */
353
+ export const INCLUDE_PARSE_TIMED_OUT = "Git didn't finish reading it in time";
354
+ /** The include parse's own limit: a loaded machine (a CI runner's parallel test files) can take seconds to start git, and
355
+ * a refusal that names a slow machine an "unreadable file" sent two releases chasing a config that was fine. */
356
+ const INCLUDE_PARSE_MS = 15_000;
361
357
  /** The include key one config file sets (lowercased, as git names it), "include" when only a plain reading finds one,
362
- * or "an unreadable file" when it can't be read as a plain file or parsed; null when it includes nothing. */
363
- function includeIn(path, gitFile = "git") {
358
+ * "an unreadable file" when it can't be read as a plain file or parsed, or INCLUDE_PARSE_TIMED_OUT when the caller's
359
+ * deadline or the parse's own limit ran out first; null when it includes nothing. Every non-null answer refuses. */
360
+ function includeIn(path, gitFile = "git", limit) {
364
361
  const text = regularFile(path);
365
362
  if (text === null)
366
363
  return "an unreadable file";
367
- const parsed = spawnSync(gitFile, ["config", "--file", path, "--no-includes", "--name-only", "--list", "-z"], { cwd: "/", encoding: "utf8", timeout: 5_000, killSignal: "SIGKILL", env: { PATH: process.env["PATH"] ?? "", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", HOME: "/nonexistent" } });
364
+ if (limit !== undefined && limit() <= 0)
365
+ return INCLUDE_PARSE_TIMED_OUT;
366
+ const parsed = spawnSync(gitFile, ["config", "--file", path, "--no-includes", "--name-only", "--list", "-z"], { cwd: "/", encoding: "utf8", timeout: limit === undefined ? INCLUDE_PARSE_MS : Math.max(1, Math.min(INCLUDE_PARSE_MS, limit())), killSignal: "SIGKILL", env: { PATH: process.env["PATH"] ?? "", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", HOME: "/nonexistent" } });
367
+ if (parsed.status === null && (parsed.signal === "SIGKILL" || parsed.error?.code === "ETIMEDOUT"))
368
+ return INCLUDE_PARSE_TIMED_OUT;
368
369
  if (parsed.status !== 0)
369
370
  return "an unreadable file";
370
371
  const named = parsed.stdout.split("\0").find(name => /^(include\.path|includeif\..*\.path)$/i.test(name));
@@ -474,7 +475,7 @@ function boundDirs(checkout, binding = {}) {
474
475
  const ADMIN_LINKED = ["index", "logs", "refs", "reftable"];
475
476
  /** What git keeps in a common directory (its path.c common_list, and reftable), linked whether or not it exists yet. */
476
477
  const COMMON_NAMES = ["branches", "gc.pid", "logs", "lost-found", "modules", "objects", "packed-refs", "reftable", "refs", "remotes", "rr-cache", "shallow", "svn", "worktrees"];
477
- function gitView(checkout, kind, binding = {}, gitFile = "git") {
478
+ function gitView(checkout, kind, binding = {}, gitFile = "git", limit) {
478
479
  const dirs = boundDirs(checkout, binding);
479
480
  const root = ensureGitViewsRoot(realpathSync(homedir()));
480
481
  const view = realpathSync(mkdtempSync(join(root, `${kind}-`)));
@@ -488,9 +489,15 @@ function gitView(checkout, kind, binding = {}, gitFile = "git") {
488
489
  throw new Error("the checkout's git config couldn't be read as a plain file");
489
490
  const configPath = join(view, "config");
490
491
  writeFileSync(configPath, config, { mode: 0o600 });
491
- const plain = { cwd: "/", encoding: "utf8", timeout: 5_000, killSignal: "SIGKILL", env: { PATH: process.env["PATH"] ?? "", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", HOME: "/nonexistent" } };
492
+ const base = { cwd: "/", encoding: "utf8", killSignal: "SIGKILL", env: { PATH: process.env["PATH"] ?? "", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", HOME: "/nonexistent" } };
493
+ // Each call's limit is worked out as it starts, from what is left of a caller's deadline.
494
+ const plainNow = () => {
495
+ if (limit !== undefined && limit() <= 0)
496
+ throw new Error("the read's deadline passed");
497
+ return { ...base, timeout: setupTimeout(limit) };
498
+ };
492
499
  if (/worktreeconfig/i.test(config)) {
493
- const flag = spawnSync(gitFile, ["config", "--file", configPath, "--no-includes", "--type=bool", "--get", "extensions.worktreeConfig"], plain);
500
+ const flag = spawnSync(gitFile, ["config", "--file", configPath, "--no-includes", "--type=bool", "--get", "extensions.worktreeConfig"], plainNow());
494
501
  if (flag.status !== 0 && flag.status !== 1)
495
502
  throw new Error("the checkout's git config couldn't be read");
496
503
  if (flag.stdout.trim() === "true") {
@@ -502,11 +509,13 @@ function gitView(checkout, kind, binding = {}, gitFile = "git") {
502
509
  config = `${config}\n${own}`;
503
510
  writeFileSync(configPath, config, { mode: 0o600 });
504
511
  }
505
- const unset = spawnSync(gitFile, ["config", "--file", configPath, "--unset-all", "extensions.worktreeConfig"], plain);
512
+ const unset = spawnSync(gitFile, ["config", "--file", configPath, "--unset-all", "extensions.worktreeConfig"], plainNow());
506
513
  if (unset.status !== 0 && unset.status !== 5)
507
514
  throw new Error("the checkout's git config couldn't be read");
508
515
  }
509
- const include = includeIn(configPath, gitFile);
516
+ const include = includeIn(configPath, gitFile, limit);
517
+ if (include === INCLUDE_PARSE_TIMED_OUT)
518
+ throw new Error(`the checkout's git config couldn't be checked: ${INCLUDE_PARSE_TIMED_OUT}; try again`);
510
519
  if (include !== null)
511
520
  throw new Error(`the checkout's git config includes another file (${include})`);
512
521
  mkdirSync(join(view, "info"));
@@ -1029,7 +1038,7 @@ export const WRITE_VIEW_FLAGS = ["-c", "worktree.useRelativePaths=false", "-c",
1029
1038
  /** Whether the checkout's .lfsconfig (read as git parses it, that file only) sets any git-lfs key that could name a
1030
1039
  * program (anything but INERT_LFS). A file that
1031
1040
  * isn't a plain one, or doesn't parse, counts as naming one. */
1032
- export function lfsExtensionIn(checkout, gitFile = "git") {
1041
+ export function lfsExtensionIn(checkout, gitFile = "git", limit) {
1033
1042
  const path = join(checkout, ".lfsconfig");
1034
1043
  try {
1035
1044
  if (!lstatSync(path).isFile())
@@ -1038,7 +1047,9 @@ export function lfsExtensionIn(checkout, gitFile = "git") {
1038
1047
  catch {
1039
1048
  return false;
1040
1049
  }
1041
- const parsed = spawnSync(gitFile, ["config", "--file", path, "--no-includes", "--name-only", "--list", "-z"], { cwd: "/", encoding: "utf8", timeout: 5_000, killSignal: "SIGKILL", env: { PATH: process.env["PATH"] ?? "", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", HOME: "/nonexistent" } });
1050
+ if (limit !== undefined && limit() <= 0)
1051
+ return true;
1052
+ const parsed = spawnSync(gitFile, ["config", "--file", path, "--no-includes", "--name-only", "--list", "-z"], { cwd: "/", encoding: "utf8", timeout: setupTimeout(limit), killSignal: "SIGKILL", env: { PATH: process.env["PATH"] ?? "", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", HOME: "/nonexistent" } });
1042
1053
  return parsed.status !== 0 || parsed.stdout.split("\0").filter(Boolean).some(name => lfsCommand(name) || ["include", "includeif"].includes(keyParts(name).section));
1043
1054
  }
1044
1055
  /** Every command runs against a view of the checkout's repository (gitView) whose config and info are copies the
@@ -1140,16 +1151,26 @@ export function checkoutGitSync(cwd, args, options = {}) {
1140
1151
  }
1141
1152
  export function viewedGitSync(cwd, args, options = {}) {
1142
1153
  const gitFile = options.git ?? "git";
1143
- if (checkoutIncludes(cwd, gitFile))
1154
+ // `deadline` (absolute, wall-clock ms): every subprocess here, setup included, gets only what is left of it, and an
1155
+ // expired deadline throws before the next one starts or after the last one ran.
1156
+ const deadline = options.deadline;
1157
+ const limit = deadline === undefined ? undefined : () => deadline - Date.now();
1158
+ const expire = () => { if (limit !== undefined && limit() <= 0)
1159
+ throw new Error("the read's deadline passed"); };
1160
+ expire();
1161
+ if (checkoutIncludes(cwd, gitFile, limit))
1144
1162
  throw new Error("the checkout's git config includes another file, or couldn't be read as a plain file");
1145
- if (lfsExtensionIn(cwd, gitFile))
1163
+ expire();
1164
+ if (lfsExtensionIn(cwd, gitFile, limit))
1146
1165
  throw new Error("the checkout's .lfsconfig sets a git-lfs key that can run a command");
1166
+ expire();
1147
1167
  // Its config and info/attributes as read once, in a folder of the plane's own: neither can change before git runs.
1148
- const view = gitView(cwd, "read", options.binding, gitFile);
1168
+ const view = gitView(cwd, "read", options.binding, gitFile, limit);
1149
1169
  try {
1150
1170
  const env = { PATH: process.env["PATH"] ?? "", HOME: process.env["HOME"] ?? "", LANG: "C", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", GIT_TERMINAL_PROMPT: "0", ...view.env };
1151
- const bounded = { cwd, env, encoding: "utf8", timeout: options.timeoutMs ?? 60_000, killSignal: "SIGKILL", maxBuffer: options.maxBuffer ?? 1_048_576, stdio: ["ignore", "pipe", "pipe"] };
1152
- const own = spawnSync(gitFile, ["--no-optional-locks", "config", "--list", "--show-scope", "-z"], bounded);
1171
+ const timeoutNow = () => { expire(); return limit === undefined ? options.timeoutMs ?? 60_000 : Math.max(1, Math.min(options.timeoutMs ?? 60_000, limit())); };
1172
+ const fixed = { cwd, env, encoding: "utf8", killSignal: "SIGKILL", maxBuffer: options.maxBuffer ?? 1_048_576, stdio: ["ignore", "pipe", "pipe"] };
1173
+ const own = spawnSync(gitFile, ["--no-optional-locks", "config", "--list", "--show-scope", "-z"], { ...fixed, timeout: timeoutNow() });
1153
1174
  if (own.status !== 0)
1154
1175
  throw new Error("its config couldn't be read");
1155
1176
  const key = commandSetting(own.stdout, { lfsChecked: true });
@@ -1170,16 +1191,18 @@ export function viewedGitSync(cwd, args, options = {}) {
1170
1191
  const argv = [...TEARDOWN_FLAGS, ...readFlags(gitFile), "-c", "diff.ignoreSubmodules=all", ...carried, ...safeArgs];
1171
1192
  if (options.buffer === true) {
1172
1193
  // stdin is the caller's own bytes when given (cat-file --batch), else closed.
1173
- const ran = spawnSync(gitFile, argv, { ...bounded, encoding: "buffer", ...(options.input === undefined ? {} : { input: options.input, stdio: ["pipe", "pipe", "pipe"] }) });
1194
+ const ran = spawnSync(gitFile, argv, { ...fixed, timeout: timeoutNow(), encoding: "buffer", ...(options.input === undefined ? {} : { input: options.input, stdio: ["pipe", "pipe", "pipe"] }) });
1174
1195
  const why = view.settle();
1175
1196
  if (why !== null)
1176
1197
  throw new Error(why);
1198
+ expire();
1177
1199
  return { status: ran.error === undefined ? ran.status : null, stdout: ran.stdout };
1178
1200
  }
1179
- const ran = spawnSync(gitFile, argv, bounded);
1201
+ const ran = spawnSync(gitFile, argv, { ...fixed, timeout: timeoutNow() });
1180
1202
  const why = view.settle();
1181
1203
  if (why !== null)
1182
1204
  throw new Error(why);
1205
+ expire();
1183
1206
  return { status: ran.error === undefined ? ran.status : null, stdout: view.restore(ran.stdout ?? "", args) };
1184
1207
  }
1185
1208
  finally {
@@ -26,6 +26,14 @@ export type UpWorkerConfig = {
26
26
  terminal: boolean;
27
27
  /** How long a start waits for a predecessor's watch lease to expire (a crashed worker's, after a restart). */
28
28
  leaseWaitMs: number;
29
+ /** Saved projects this start couldn't reach (a folder that didn't answer, a drive not connected): looked at again
30
+ * while it runs, and connected once they answer, on the same saved-list authority the start had. */
31
+ deferredRepos?: string[];
32
+ /** Each of those lines' removal count when this start read the saved list: a different count at binding means it was
33
+ * removed since (even if saved again), and the start's authority for it is gone. */
34
+ deferredGenerations?: Record<string, number>;
35
+ /** The saved line each of `repos` was proved from: a project stays while its line is saved, whatever else answers. */
36
+ repoLines?: Record<string, string[]>;
29
37
  };
30
38
  export type UpRoleStart = {
31
39
  type: "start";
@@ -67,6 +75,11 @@ export type RoleToParent = {
67
75
  } | {
68
76
  type: "repos";
69
77
  repos: string[];
78
+ }
79
+ /** The saved projects the worker still holds start authority for (it only ever shrinks: connected or removed). */
80
+ | {
81
+ type: "deferred";
82
+ repos: string[];
70
83
  } | {
71
84
  type: "failed";
72
85
  reason: "port-busy" | "start";
@@ -53,6 +53,7 @@ export function roleMessage(value, role) {
53
53
  return { type: "ready", role, build: value["build"], pid: value["pid"], url: value["url"], port: value["port"] };
54
54
  case "log": return text(value["line"]) ? { type: "log", line: value["line"] } : null;
55
55
  case "repos": return role === "worker" && texts(value["repos"]) ? { type: "repos", repos: [...value["repos"]] } : null;
56
+ case "deferred": return role === "worker" && texts(value["repos"]) ? { type: "deferred", repos: [...value["repos"]] } : null;
56
57
  case "failed": return (value["reason"] === "port-busy" || value["reason"] === "start") && text(value["detail"]) ? { type: "failed", reason: value["reason"], detail: value["detail"] } : null;
57
58
  case "fatal": return text(value["detail"]) ? { type: "fatal", detail: value["detail"] } : null;
58
59
  case "sign-in-link": return role === "console" && whole(value["id"]) && orNull(value["link"], (one) => nonEmpty(one, 2_048)) ? { type: "sign-in-link", id: value["id"], link: value["link"] } : null;
@@ -78,12 +79,15 @@ export function startMessage(value) {
78
79
  return { ...base, role: "console", config: config };
79
80
  }
80
81
  if (value["role"] === "worker") {
81
- const keys = ["runner", "token", "repos", "pinnedRepos", "projectRoots", "registryPath", "pool", "forMs", "publicUrl", "stopGraceMs", "verbose", "terminal", "leaseWaitMs"];
82
+ const keys = ["runner", "token", "repos", "pinnedRepos", "projectRoots", "registryPath", "pool", "forMs", "publicUrl", "stopGraceMs", "verbose", "terminal", "leaseWaitMs", "deferredRepos", "deferredGenerations", "repoLines"];
82
83
  if (Object.keys(config).some(key => !keys.includes(key)))
83
84
  return null;
84
85
  if (!nonEmpty(config["runner"], 256) || !nonEmpty(config["token"], 1_024) || !texts(config["repos"]) || !texts(config["pinnedRepos"]) || !texts(config["projectRoots"]) ||
85
86
  !nonEmpty(config["registryPath"], 4_096) || !nonEmpty(config["pool"], 4_096) || !orNull(config["forMs"], (one) => whole(one, 1)) ||
86
- !orNull(config["publicUrl"], (one) => nonEmpty(one, 2_048)) || !whole(config["stopGraceMs"], 0) || typeof config["verbose"] !== "boolean" || typeof config["terminal"] !== "boolean" || !whole(config["leaseWaitMs"], 0))
87
+ !orNull(config["publicUrl"], (one) => nonEmpty(one, 2_048)) || !whole(config["stopGraceMs"], 0) || typeof config["verbose"] !== "boolean" || typeof config["terminal"] !== "boolean" || !whole(config["leaseWaitMs"], 0) ||
88
+ (config["deferredRepos"] !== undefined && !texts(config["deferredRepos"])) ||
89
+ (config["deferredGenerations"] !== undefined && !(isObject(config["deferredGenerations"]) && Object.values(config["deferredGenerations"]).every(one => whole(one, -1, Number.MAX_SAFE_INTEGER)))) ||
90
+ (config["repoLines"] !== undefined && !(isObject(config["repoLines"]) && Object.values(config["repoLines"]).every(one => texts(one)))))
87
91
  return null;
88
92
  return { ...base, role: "worker", config: config };
89
93
  }
@@ -0,0 +1,25 @@
1
+ {
2
+ "id": "catalog",
3
+ "kind": "build",
4
+ "title": "Add slug and catalog",
5
+ "goal": "Add slug.js exporting slug(text): trim and lowercase ASCII text and collapse every run of non-alphanumeric characters to one hyphen, with no hyphen at either end. Add catalog.js exporting catalog(titles): return { title, slug } records in order, and throw an Error when two titles give the same slug.",
6
+ "touches": [
7
+ "slug.js",
8
+ "catalog.js",
9
+ "test/catalog.test.mjs"
10
+ ],
11
+ "pin": {
12
+ "provider": "claude",
13
+ "model": "claude-sonnet-5-5"
14
+ },
15
+ "maxDispatches": 2,
16
+ "timeoutMinutes": 15,
17
+ "grade": {
18
+ "files": [
19
+ "slug.js",
20
+ "catalog.js"
21
+ ]
22
+ },
23
+ "expected": "pass",
24
+ "validatedBy": null
25
+ }
@@ -0,0 +1,3 @@
1
+ # Eval fixture
2
+
3
+ Run `npm test`. Change only what the task asks for.
@@ -0,0 +1,8 @@
1
+ {
2
+ "name": "eval-fixture",
3
+ "private": true,
4
+ "type": "module",
5
+ "scripts": {
6
+ "test": "node --test"
7
+ }
8
+ }
@@ -0,0 +1,4 @@
1
+ import test from 'node:test';
2
+ import assert from 'node:assert/strict';
3
+
4
+ test('the fixture runs', () => assert.ok(true));
@@ -0,0 +1,23 @@
1
+ // The hidden check's own answers (zero dependencies). Run only by the plane's oracle process, never inside a world or
2
+ // the fence: `inputs` draws fresh inputs from a source seeded by each grading's nonce, and `expected` is the answer
3
+ // the oracle computes itself for each one.
4
+ export const files = ["slug.js", "catalog.js"];
5
+ const CHARS = "abcXYZ019 &!-_.,";
6
+ export function inputs(random) {
7
+ const int = (low, high) => low + Math.floor(random() * (high - low + 1));
8
+ const word = () => { let out = ""; const length = int(1, 12); for (let i = 0; i < length; i++) out += CHARS[int(0, CHARS.length - 1)]; return /[a-z0-9]/i.test(out) ? out : out + "q"; };
9
+ const out = [];
10
+ for (let i = 0; i < 8; i++) out.push({ kind: "slug", text: word() });
11
+ for (let i = 0; i < 6; i++) {
12
+ const titles = Array.from({ length: int(0, 4) }, word);
13
+ if (i % 3 === 0 && titles.length > 0) titles.push(titles[0].toUpperCase() + "!");
14
+ out.push({ kind: "catalog", titles });
15
+ }
16
+ return out;
17
+ }
18
+ const slug = text => text.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
19
+ export function expected(input) {
20
+ if (input.kind === "slug") return { value: slug(input.text) };
21
+ const records = input.titles.map(title => ({ title, slug: slug(title) }));
22
+ return new Set(records.map(one => one.slug)).size === records.length ? { value: records } : { threw: true };
23
+ }
@@ -0,0 +1,10 @@
1
+ // The graded program's launcher. The oracle copies this file from the installed case (never the build's copy) and
2
+ // runs it inside the fence with the nonce and inputs on stdin; only a line naming that nonce with the right outputs
3
+ // passes. Nothing here is visible to the agent while it builds.
4
+ import { readFileSync } from "node:fs";
5
+ const { nonce, inputs } = JSON.parse(readFileSync(0, "utf8"));
6
+ const attempt = run => { try { return run(); } catch { return { threw: true }; } };
7
+ const { slug } = await import("./slug.js");
8
+ const { catalog } = await import("./catalog.js");
9
+ const outputs = inputs.map(input => attempt(() => ({ value: input.kind === "slug" ? slug(input.text) : catalog(input.titles) })));
10
+ process.stdout.write(JSON.stringify({ nonce, outputs }) + "\n");
@@ -0,0 +1,9 @@
1
+ # catalog
2
+
3
+ From the real-model pilot's fixed assignments (scripts/fixtures/pilot-scenarios.mjs, `catalog`, "multiple files"): two new modules, one using the other, with a duplicate rule that is easy to get wrong after case folding.
4
+
5
+ Expected: pass.
6
+
7
+ ## Status
8
+
9
+ Draft. The expected outcome above awaits a person's validation (`validatedBy` in case.json); until then this case counts in no canonical campaign.
@@ -0,0 +1,23 @@
1
+ {
2
+ "id": "csv-quoting",
3
+ "kind": "build",
4
+ "title": "Quote CSV fields",
5
+ "goal": "Revise csv(rows) in csv.js to support comma, quote and newline characters in fields: quote such fields and double any quote inside them. Join fields with commas and records with CRLF, and end non-empty output with a final CRLF. An empty list of rows returns an empty string. Keep the csv export.",
6
+ "touches": [
7
+ "csv.js",
8
+ "test/csv.test.mjs"
9
+ ],
10
+ "pin": {
11
+ "provider": "claude",
12
+ "model": "claude-sonnet-5-5"
13
+ },
14
+ "maxDispatches": 2,
15
+ "timeoutMinutes": 15,
16
+ "grade": {
17
+ "files": [
18
+ "csv.js"
19
+ ]
20
+ },
21
+ "expected": "pass",
22
+ "validatedBy": null
23
+ }
@@ -0,0 +1,3 @@
1
+ # Eval fixture
2
+
3
+ Run `npm test`. Change only what the task asks for.
@@ -0,0 +1 @@
1
+ export const csv = rows => rows.map(r => r.join(',')).join('\n');
@@ -0,0 +1,8 @@
1
+ {
2
+ "name": "eval-fixture",
3
+ "private": true,
4
+ "type": "module",
5
+ "scripts": {
6
+ "test": "node --test"
7
+ }
8
+ }
@@ -0,0 +1,5 @@
1
+ import test from 'node:test';
2
+ import assert from 'node:assert/strict';
3
+ import { csv } from '../csv.js';
4
+
5
+ test('no rows', () => assert.equal(csv([]), ''));
@@ -0,0 +1,14 @@
1
+ // The hidden check's own answers (zero dependencies). Run only by the plane's oracle process, never inside a world or
2
+ // the fence: `inputs` draws fresh inputs from a source seeded by each grading's nonce, and `expected` is the answer
3
+ // the oracle computes itself for each one.
4
+ export const files = ["csv.js"];
5
+ const PARTS = ["a", "b c", ",", '"', "\n", "x", "0", ""];
6
+ export function inputs(random) {
7
+ const int = (low, high) => low + Math.floor(random() * (high - low + 1));
8
+ const field = () => random() < 0.15 ? int(0, 99) : Array.from({ length: int(0, 3) }, () => PARTS[int(0, PARTS.length - 1)]).join("");
9
+ return Array.from({ length: 10 }, () => ({ rows: Array.from({ length: int(0, 3) }, () => Array.from({ length: int(1, 4) }, field)) }));
10
+ }
11
+ const cell = value => { const text = String(value); return /[",\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; };
12
+ export function expected({ rows }) {
13
+ return { value: rows.length === 0 ? "" : rows.map(row => row.map(cell).join(",")).join("\r\n") + "\r\n" };
14
+ }
@@ -0,0 +1,9 @@
1
+ // The graded program's launcher. The oracle copies this file from the installed case (never the build's copy) and
2
+ // runs it inside the fence with the nonce and inputs on stdin; only a line naming that nonce with the right outputs
3
+ // passes. Nothing here is visible to the agent while it builds.
4
+ import { readFileSync } from "node:fs";
5
+ const { nonce, inputs } = JSON.parse(readFileSync(0, "utf8"));
6
+ const attempt = run => { try { return run(); } catch { return { threw: true }; } };
7
+ const { csv } = await import("./csv.js");
8
+ const outputs = inputs.map(({ rows }) => attempt(() => ({ value: csv(rows) })));
9
+ process.stdout.write(JSON.stringify({ nonce, outputs }) + "\n");