toolroll 0.9.81 → 0.9.82

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/dist/cli.js +1 -0
  3. package/dist/commit-extract.d.ts +18 -0
  4. package/dist/commit-extract.js +39 -1
  5. package/dist/converse.d.ts +12 -0
  6. package/dist/converse.js +18 -2
  7. package/dist/deploy-gate.js +8 -0
  8. package/dist/eval-gate.d.ts +51 -0
  9. package/dist/eval-gate.js +199 -0
  10. package/dist/eval-hooks.d.ts +39 -0
  11. package/dist/eval-hooks.js +477 -0
  12. package/dist/eval-oracle-main.d.ts +1 -0
  13. package/dist/eval-oracle-main.js +31 -0
  14. package/dist/eval-oracle.d.ts +60 -0
  15. package/dist/eval-oracle.js +217 -0
  16. package/dist/eval-world.d.ts +17 -0
  17. package/dist/eval-world.js +97 -0
  18. package/dist/evals.d.ts +299 -0
  19. package/dist/evals.js +964 -0
  20. package/dist/exec.js +15 -1
  21. package/dist/flow-draft.js +4 -0
  22. package/dist/flow-sort.js +4 -0
  23. package/dist/github-jobs.d.ts +25 -0
  24. package/dist/github-jobs.js +22 -0
  25. package/dist/invoke.d.ts +1 -1
  26. package/dist/invoke.js +14 -0
  27. package/dist/lead.js +4 -1
  28. package/dist/memory-pass.js +4 -0
  29. package/dist/nightly-check.d.ts +4 -1
  30. package/dist/nightly-check.js +5 -2
  31. package/dist/nightly-github.d.ts +316 -0
  32. package/dist/nightly-github.js +1022 -0
  33. package/dist/operate.d.ts +91 -1
  34. package/dist/operate.js +158 -1
  35. package/dist/project-knowledge.d.ts +4 -4
  36. package/dist/provider.d.ts +3 -0
  37. package/dist/provider.js +9 -2
  38. package/dist/release-adapters.d.ts +21 -21
  39. package/dist/release-adapters.js +52 -19
  40. package/dist/release-auto.js +26 -0
  41. package/dist/release-card.js +3 -0
  42. package/dist/release-cli.js +59 -3
  43. package/dist/release-view.d.ts +3 -0
  44. package/dist/release-view.js +2 -0
  45. package/dist/release.d.ts +16 -0
  46. package/dist/release.js +16 -0
  47. package/dist/reported-usage.d.ts +69 -0
  48. package/dist/reported-usage.js +97 -0
  49. package/dist/server/chat.js +4 -0
  50. package/dist/server/render-tasks.js +1 -0
  51. package/dist/server/tasks.js +8 -1
  52. package/dist/store.d.ts +109 -1
  53. package/dist/store.js +394 -4
  54. package/dist/subagents.js +4 -0
  55. package/dist/subscription-chat.d.ts +2 -0
  56. package/dist/subscription-chat.js +52 -13
  57. package/dist/task-sizing.js +4 -0
  58. package/dist/teardown-git.d.ts +9 -2
  59. package/dist/teardown-git.js +56 -33
  60. package/evals/cases/catalog/case.json +25 -0
  61. package/evals/cases/catalog/fixture/README.md +3 -0
  62. package/evals/cases/catalog/fixture/package.json +8 -0
  63. package/evals/cases/catalog/fixture/test/catalog.test.mjs +4 -0
  64. package/evals/cases/catalog/oracle.mjs +23 -0
  65. package/evals/cases/catalog/probe.mjs +10 -0
  66. package/evals/cases/catalog/source.md +9 -0
  67. package/evals/cases/csv-quoting/case.json +23 -0
  68. package/evals/cases/csv-quoting/fixture/README.md +3 -0
  69. package/evals/cases/csv-quoting/fixture/csv.js +1 -0
  70. package/evals/cases/csv-quoting/fixture/package.json +8 -0
  71. package/evals/cases/csv-quoting/fixture/test/csv.test.mjs +5 -0
  72. package/evals/cases/csv-quoting/oracle.mjs +14 -0
  73. package/evals/cases/csv-quoting/probe.mjs +9 -0
  74. package/evals/cases/csv-quoting/source.md +9 -0
  75. package/evals/cases/dedupe/case.json +23 -0
  76. package/evals/cases/dedupe/fixture/README.md +3 -0
  77. package/evals/cases/dedupe/fixture/dedupe.js +1 -0
  78. package/evals/cases/dedupe/fixture/package.json +8 -0
  79. package/evals/cases/dedupe/fixture/test/dedupe.test.mjs +5 -0
  80. package/evals/cases/dedupe/oracle.mjs +16 -0
  81. package/evals/cases/dedupe/probe.mjs +13 -0
  82. package/evals/cases/dedupe/source.md +9 -0
  83. package/evals/cases/endpoint/case.json +25 -0
  84. package/evals/cases/endpoint/fixture/README.md +3 -0
  85. package/evals/cases/endpoint/fixture/package.json +8 -0
  86. package/evals/cases/endpoint/fixture/test/endpoint.test.mjs +4 -0
  87. package/evals/cases/endpoint/oracle.mjs +27 -0
  88. package/evals/cases/endpoint/probe.mjs +10 -0
  89. package/evals/cases/endpoint/source.md +11 -0
  90. package/evals/cases/math-divide/case.json +23 -0
  91. package/evals/cases/math-divide/fixture/README.md +3 -0
  92. package/evals/cases/math-divide/fixture/package.json +8 -0
  93. package/evals/cases/math-divide/fixture/src/math.js +1 -0
  94. package/evals/cases/math-divide/fixture/test/math.test.js +5 -0
  95. package/evals/cases/math-divide/oracle.mjs +13 -0
  96. package/evals/cases/math-divide/probe.mjs +9 -0
  97. package/evals/cases/math-divide/source.md +9 -0
  98. package/evals/cases/math-multiply/case.json +23 -0
  99. package/evals/cases/math-multiply/fixture/README.md +3 -0
  100. package/evals/cases/math-multiply/fixture/package.json +8 -0
  101. package/evals/cases/math-multiply/fixture/src/math.js +2 -0
  102. package/evals/cases/math-multiply/fixture/test/math.test.js +5 -0
  103. package/evals/cases/math-multiply/oracle.mjs +12 -0
  104. package/evals/cases/math-multiply/probe.mjs +9 -0
  105. package/evals/cases/math-multiply/source.md +9 -0
  106. package/evals/cases/math-subtract/case.json +23 -0
  107. package/evals/cases/math-subtract/fixture/README.md +3 -0
  108. package/evals/cases/math-subtract/fixture/package.json +8 -0
  109. package/evals/cases/math-subtract/fixture/src/math.js +1 -0
  110. package/evals/cases/math-subtract/fixture/test/math.test.js +5 -0
  111. package/evals/cases/math-subtract/oracle.mjs +12 -0
  112. package/evals/cases/math-subtract/probe.mjs +9 -0
  113. package/evals/cases/math-subtract/source.md +9 -0
  114. package/evals/cases/merge-config/case.json +23 -0
  115. package/evals/cases/merge-config/fixture/README.md +3 -0
  116. package/evals/cases/merge-config/fixture/config.js +1 -0
  117. package/evals/cases/merge-config/fixture/package.json +8 -0
  118. package/evals/cases/merge-config/fixture/test/config.test.mjs +5 -0
  119. package/evals/cases/merge-config/oracle.mjs +28 -0
  120. package/evals/cases/merge-config/probe.mjs +15 -0
  121. package/evals/cases/merge-config/source.md +9 -0
  122. package/evals/cases/paginate/case.json +23 -0
  123. package/evals/cases/paginate/fixture/README.md +3 -0
  124. package/evals/cases/paginate/fixture/package.json +8 -0
  125. package/evals/cases/paginate/fixture/paginate.js +1 -0
  126. package/evals/cases/paginate/fixture/test/paginate.test.mjs +5 -0
  127. package/evals/cases/paginate/oracle.mjs +18 -0
  128. package/evals/cases/paginate/probe.mjs +13 -0
  129. package/evals/cases/paginate/source.md +9 -0
  130. package/evals/cases/round2-asks/case.json +24 -0
  131. package/evals/cases/round2-asks/fixture/README.md +3 -0
  132. package/evals/cases/round2-asks/fixture/package.json +8 -0
  133. package/evals/cases/round2-asks/fixture/src/math.js +1 -0
  134. package/evals/cases/round2-asks/fixture/test/math.test.js +5 -0
  135. package/evals/cases/round2-asks/source.md +11 -0
  136. package/evals/manifest.json +15 -0
  137. package/package.json +2 -1
package/CHANGELOG.md CHANGED
@@ -1,5 +1,26 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.9.82 — 2026-10-31
4
+
5
+ - **A failing nightly holds your releases.** Turn it on with
6
+ `toolroll release nightly --github` (your password). When a lane of your
7
+ project's GitHub nightly fails on main, every deploy, the merge and the tag
8
+ of a release wait until a later nightly passes that lane, and Needs you
9
+ says which lane and why. A release whose own pull request re-ran that lane
10
+ after the failure, with the same CI files, goes through. A nightly that
11
+ can't be read gives a warning, never a pass.
12
+ - **Every run and chat turn records the model and cost the provider
13
+ reported:** the exact model id, provider turns, cache-aware tokens, and
14
+ cost, with API dollars, subscription usage and unknown kept apart (unknown
15
+ is never shown as zero).
16
+ - **Evals (groundwork).** `toolroll evals` runs repeatable test tasks in
17
+ throwaway worlds, graded by a separate checker process the agent can't
18
+ reach. Ten starter tasks ship as drafts; real comparison runs stay off
19
+ until they're validated.
20
+ - **CI:** required checks report under their exact names on every pull
21
+ request; a slow git config read now says it timed out instead of blaming
22
+ the project's config.
23
+
3
24
  ## 0.9.81 — 2026-10-30
4
25
 
5
26
  - **The release proof works after the merge.** GitHub empties a CI run's
package/dist/cli.js CHANGED
@@ -108,6 +108,7 @@ export function parseArgs(argv) {
108
108
  /** The commands that operate the queue rather than report on the world. */
109
109
  export const OPERATE_COMMANDS = new Set([
110
110
  "up",
111
+ "evals",
111
112
  // Only `lead token` reaches here; the central service's lead verbs answer first.
112
113
  "lead",
113
114
  "onboard",
@@ -71,6 +71,11 @@ export type Verified<T> = {
71
71
  export type GitRunner = {
72
72
  git?: string;
73
73
  env?: NodeJS.ProcessEnv;
74
+ /** Each git call's time limit (default five minutes). */
75
+ timeoutMs?: number;
76
+ /** An absolute wall-clock deadline (ms): every git subprocess, setup included, gets only what is left of it, and a
77
+ * read past it fails (teardown-git.ts viewedGitSync). */
78
+ deadline?: number;
74
79
  };
75
80
  /** One commit, hashed here and parsed from its own bytes. */
76
81
  export declare function readVerifiedCommit(gitDir: string, sha: string): Verified<VerifiedCommit>;
@@ -114,4 +119,17 @@ export declare function verifiedRange(gitDir: string, from: string, to: string,
114
119
  max?: number;
115
120
  reach?: number;
116
121
  }): Verified<VerifiedCommit[]>;
122
+ /**
123
+ * Whether `until` lies on `from`'s first-parent history, within a deadline (nightly-github.ts): "yes" when the walk
124
+ * reaches it; "no" only when it is PROVEN not to — `until` is itself a readable, verified commit and every commit from
125
+ * `from` back to the root was read and verified without meeting it; "unknown" for anything less certain (an object
126
+ * missing or unreadable, `max` commits walked, the deadline reached). Every git call is bounded by what is left of
127
+ * the deadline, at most `callMs`.
128
+ */
129
+ export declare function verifiedFirstParentAncestry(gitDir: string, from: string, until: string, options: {
130
+ deadline: number;
131
+ now?: () => number;
132
+ max?: number;
133
+ callMs?: number;
134
+ }): "yes" | "no" | "unknown";
117
135
  export {};
@@ -212,7 +212,7 @@ function protectedGit(gitDir, runner, extra) {
212
212
  if (checkout === null)
213
213
  return { status: null, stdout: Buffer.alloc(0), refused: "it isn't a checkout's .git folder, which this git can only read through a view" };
214
214
  try {
215
- const ran = viewedGitSync(checkout, [...base, ...args], { buffer: true, timeoutMs: 300_000, maxBuffer: MAX_BYTES, ...(input === undefined ? {} : { input }), ...(runner.git === undefined ? {} : { git: runner.git }) });
215
+ const ran = viewedGitSync(checkout, [...base, ...args], { buffer: true, timeoutMs: runner.timeoutMs ?? 300_000, ...(runner.deadline === undefined ? {} : { deadline: runner.deadline }), maxBuffer: MAX_BYTES, ...(input === undefined ? {} : { input }), ...(runner.git === undefined ? {} : { git: runner.git }) });
216
216
  return { status: ran.status, stdout: ran.stdout, refused: null };
217
217
  }
218
218
  catch (error) {
@@ -567,3 +567,41 @@ export function verifiedRange(gitDir, from, to, options = {}) {
567
567
  const unreached = wanted.find(one => !reached.has(one));
568
568
  return unreached === undefined ? { ok: true, value: commits } : { ok: false, message: `${unreached.slice(0, 7)} isn't verified as part of the last release's history.` };
569
569
  }
570
+ /**
571
+ * Whether `until` lies on `from`'s first-parent history, within a deadline (nightly-github.ts): "yes" when the walk
572
+ * reaches it; "no" only when it is PROVEN not to — `until` is itself a readable, verified commit and every commit from
573
+ * `from` back to the root was read and verified without meeting it; "unknown" for anything less certain (an object
574
+ * missing or unreadable, `max` commits walked, the deadline reached). Every git call is bounded by what is left of
575
+ * the deadline, at most `callMs`.
576
+ */
577
+ export function verifiedFirstParentAncestry(gitDir, from, until, options) {
578
+ const now = options.now ?? Date.now;
579
+ const left = () => options.deadline - now();
580
+ // Every answer is checked against the deadline as it is given: nothing past it is ever "yes" or "no".
581
+ const answer = (value) => left() > 0 ? value : "unknown";
582
+ if (left() <= 0)
583
+ return "unknown";
584
+ // The absolute deadline goes down to every subprocess (setup included); each call is also held to `callMs`.
585
+ const reader = verifiedReader(gitDir, { timeoutMs: options.callMs ?? 10_000, deadline: options.deadline });
586
+ if (!reader.ok)
587
+ return "unknown";
588
+ if (from === until)
589
+ return readCommitWith(reader.value, until).ok ? answer("yes") : "unknown";
590
+ if (left() <= 0 || !readCommitWith(reader.value, until).ok)
591
+ return "unknown";
592
+ let at = from;
593
+ for (let walked = 0; walked < (options.max ?? 20_000); walked++) {
594
+ if (left() <= 0)
595
+ return "unknown";
596
+ const commit = readCommitWith(reader.value, at);
597
+ if (!commit.ok)
598
+ return "unknown";
599
+ const parent = commit.value.parents[0];
600
+ if (parent === undefined)
601
+ return answer("no");
602
+ if (parent === until)
603
+ return answer("yes");
604
+ at = parent;
605
+ }
606
+ return "unknown";
607
+ }
@@ -132,6 +132,9 @@ export type ProviderAnswer = {
132
132
  /** OpenRouter reports the actual charge; settlement takes the HIGHER of
133
133
  * this and the pinned math — the ledger never undercounts. */
134
134
  reportedCostMicrousd: number | null;
135
+ /** v140: the exact model the response body names, and the cost it reported in dollars (absent or null: unreported). */
136
+ modelReported?: string | null;
137
+ reportedCostUsd?: number | null;
135
138
  };
136
139
  /** Strict parse of the PROVIDER response body (layer one). */
137
140
  /** A token count is a non-negative safe integer under a billion (mate arc,
@@ -184,6 +187,15 @@ export type LeadProviderAnswer = {
184
187
  reportedCostMicrousd: number | null;
185
188
  cacheWriteTokens?: number;
186
189
  cacheReadTokens?: number;
190
+ /** v140 (docs/plans/evals-nightly.md B1), each from the provider's own report and absent or null when it gave none
191
+ * (never 0): the exact model it says answered, its own turn count (`num_turns`), and the cost it reported in dollars.
192
+ * `reportedCostMicrousd` stays API money only; a subscription harness's reported cost is `reportedCostUsd` alone. */
193
+ modelReported?: string | null;
194
+ providerTurns?: number | null;
195
+ reportedCostUsd?: number | null;
196
+ /** v141: how a subscription harness's own events said this request was billed (its init's key source); null or
197
+ * absent when they didn't say (unknown money, never assumed to be the plan). */
198
+ billingReported?: "subscription" | "api-key" | null;
187
199
  };
188
200
  /** Anthropic prices a cache write at 1.25x and a cache read at 0.1x the input price (5-minute cache). */
189
201
  export declare const CACHE_WRITE_RATIO = 1.25;
package/dist/converse.js CHANGED
@@ -19,10 +19,12 @@
19
19
  * micro-dollars per token, rounded up. A model without a pinned price
20
20
  * cannot be spoken to — the reservation math would be fiction.
21
21
  */
22
+ import { takeEvalDispatch } from "./eval-gate.js";
22
23
  import { TASK_TEXT_LIMITS } from "./task-text.js";
23
24
  import { createHash } from "node:crypto";
24
25
  import { hasForbiddenControls, hasDisguisedText } from "./decision.js";
25
26
  import { parseAcceptanceCriteria } from "./scope.js";
27
+ import { reportedCostUsdOf, reportedModelOf } from "./reported-usage.js";
26
28
  // ---------------------------------------------------------------- limits
27
29
  export const WRAPPER_CAP_BYTES = 262_144; // the provider response body
28
30
  export const ENVELOPE_CAP_BYTES = 65_536; // the assistant text inside it
@@ -481,6 +483,7 @@ export function parseLeadProviderWrapper(provider, bytes) {
481
483
  if (write === null || read === null)
482
484
  return { ok: false, problem: "no-usage" };
483
485
  return { ok: true, answer: { text, calls, tokensIn: usage.tokensIn + (write ?? 0) + (read ?? 0), tokensOut: usage.tokensOut, reportedCostMicrousd: null,
486
+ modelReported: reportedModelOf(body["model"]), providerTurns: null, reportedCostUsd: null,
484
487
  ...(write === undefined ? {} : { cacheWriteTokens: write }), ...(read === undefined ? {} : { cacheReadTokens: read }) } };
485
488
  }
486
489
  const choices = body["choices"];
@@ -527,7 +530,8 @@ export function parseLeadProviderWrapper(provider, bytes) {
527
530
  return { ok: false, problem: "bad-cost" };
528
531
  reportedCostMicrousd = micro;
529
532
  }
530
- return { ok: true, answer: { text, calls, tokensIn: usage.tokensIn, tokensOut: usage.tokensOut, reportedCostMicrousd } };
533
+ return { ok: true, answer: { text, calls, tokensIn: usage.tokensIn, tokensOut: usage.tokensOut, reportedCostMicrousd,
534
+ modelReported: reportedModelOf(body["model"]), providerTurns: null, reportedCostUsd: reportedCostUsdOf(rawCost) } };
531
535
  }
532
536
  /** The lead's request: system contract, the data document as the first
533
537
  * operator message, the history in provider-native shape, the tools. */
@@ -593,6 +597,10 @@ export function composeLeadRequest(args) {
593
597
  }
594
598
  /** The lead's network call: the same transport posture as fleet chat's, the tool-capable parser at the end. */
595
599
  export async function performLeadRequest(request, provider, signal, fetcher = fetch) {
600
+ // An eval world (eval-gate.ts): each request takes one of the attempt's dispatch slots. Past the limit nothing is sent.
601
+ const evalSlot = takeEvalDispatch(`lead:${provider}`);
602
+ if (evalSlot.active && !evalSlot.ok)
603
+ return { ok: false, problem: "status-eval-dispatch-limit" };
596
604
  let response;
597
605
  try {
598
606
  response = await fetcher(request.url, { method: "POST", headers: request.headers, body: request.body, redirect: "error", signal });
@@ -634,7 +642,7 @@ export function parseProviderWrapper(provider, bytes) {
634
642
  const tokensOut = usage === undefined ? NaN : Number(usage["output_tokens"]);
635
643
  if (!tokenCount(tokensIn) || !tokenCount(tokensOut))
636
644
  return { ok: false, problem: "no-usage" };
637
- return { ok: true, answer: { text: block["text"], tokensIn, tokensOut, reportedCostMicrousd: null } };
645
+ return { ok: true, answer: { text: block["text"], tokensIn, tokensOut, reportedCostMicrousd: null, modelReported: reportedModelOf(body["model"]), reportedCostUsd: null } };
638
646
  }
639
647
  // openrouter-api (OpenAI-compatible)
640
648
  const choices = body["choices"];
@@ -660,6 +668,9 @@ export function parseProviderWrapper(provider, bytes) {
660
668
  tokensIn,
661
669
  tokensOut,
662
670
  reportedCostMicrousd: Number.isFinite(reportedCost) && reportedCost >= 0 ? Math.ceil(reportedCost * 1_000_000) : null,
671
+ modelReported: reportedModelOf(body["model"]),
672
+ // The body's own number only: Number(null) would read an unreported cost as 0.
673
+ reportedCostUsd: usage === undefined ? null : reportedCostUsdOf(usage["cost"]),
663
674
  },
664
675
  };
665
676
  }
@@ -801,6 +812,11 @@ export async function readCappedBody(response, cap) {
801
812
  * wall clock, streamed cap, wrapper strictness.
802
813
  */
803
814
  export async function performChatRequest(args, fetcher = fetch) {
815
+ // An eval world (eval-gate.ts, Codex round 1 #3): fleet chat's request takes a dispatch slot too; past the limit, or
816
+ // with a gate that can't be read, nothing is sent.
817
+ const evalSlot = takeEvalDispatch(`chat:${args.provider}`);
818
+ if (evalSlot.active && !evalSlot.ok)
819
+ return { ok: false, problem: "status-eval-dispatch-limit" };
804
820
  const request = composeRequest(args);
805
821
  let response;
806
822
  try {
@@ -9,6 +9,7 @@ import { assignmentOf } from "./assignment.js";
9
9
  import { liveReleaseOf, personApprovalOf, releaseApprovalOf } from "./scope.js";
10
10
  import { verificationEvidence } from "./verification-evidence.js";
11
11
  import { releaseCiRunningOf } from "./release-ci-hold.js";
12
+ import { nightlyDebtHold } from "./nightly-github.js";
12
13
  /** Whether run `runId` is a deployable candidate now, read only from the plane's records and this runtime's code. */
13
14
  export function deploymentGate(store, input) {
14
15
  const refuse = (message) => ({ ok: false, message });
@@ -40,6 +41,13 @@ export function deploymentGate(store, input) {
40
41
  return refuse(`This release can't deploy: ${release.why}.`);
41
42
  const anchorPerson = release.applies && release.ok
42
43
  ? (input.releaseFlow === true ? releaseApprovalOf(store, release.authority.anchorTask, input.now) : personApprovalOf(store, release.authority.anchorTask)) : null;
44
+ // GitHub's nightly lane debt (nightly-github.ts) holds every deploy of the project, a person's approval or not,
45
+ // unless this exact candidate re-ran that lane after the failure and passed (an exemption read at the boundary).
46
+ if (ref.repo !== null) {
47
+ const nightly = nightlyDebtHold(store, { repo: ref.repo, taskId: ref.externalId, candidate: scope?.candidate ?? head, phase: "deploy", release: release.applies, now: input.now });
48
+ if (nightly !== null)
49
+ return refuse(nightly.why);
50
+ }
43
51
  const person = anchorPerson ?? personApprovalOf(store, ref.externalId);
44
52
  if (!person.ok)
45
53
  return refuse(`A person has to approve this candidate's scope: ${person.why}.`);
@@ -0,0 +1,51 @@
1
+ export declare const EVAL_GATE_ENV = "TOOLROLL_EVAL_GATE";
2
+ /** The gate's terms: how many provider dispatches the attempt may make. */
3
+ export type EvalGateConfig = {
4
+ v: 2;
5
+ max: number;
6
+ };
7
+ /** One slot a dispatch took. */
8
+ export type EvalGateSlot = {
9
+ n: number;
10
+ kind: string;
11
+ pid: number;
12
+ at: string;
13
+ };
14
+ export type EvalDispatch = {
15
+ active: false;
16
+ } | {
17
+ active: true;
18
+ ok: true;
19
+ n: number;
20
+ } | {
21
+ active: true;
22
+ ok: false;
23
+ why: string;
24
+ };
25
+ /** At most `limit` bodies at once; the others start in the order they asked, as each running one ends (provider-gate
26
+ * limiter). */
27
+ export declare function limiter(limit: number): <T>(body: () => Promise<T>) => Promise<T>;
28
+ /** Read a small regular file without following a link; null when it isn't one, is too big or can't be read. */
29
+ export declare function readNoFollow(path: string, maxBytes: number): Buffer | null;
30
+ /** Make a world's gate in `dir` (a fresh folder the runner owns): its terms, and an empty slots folder. Refuses a folder
31
+ * that already holds a gate. */
32
+ export declare function openEvalGate(dir: string, config: EvalGateConfig): void;
33
+ /** The world's gate, as this process's environment names it: null outside an eval world, its terms, or why it can't
34
+ * be read (which refuses every dispatch). */
35
+ export declare function evalWorldGate(env?: Readonly<Record<string, string | undefined>>): {
36
+ dir: string;
37
+ config: EvalGateConfig;
38
+ } | {
39
+ dir: string;
40
+ problem: string;
41
+ } | null;
42
+ /** Take one dispatch slot of this eval world, before a provider is called. Outside an eval world: `{ active: false }`. */
43
+ export declare function takeEvalDispatch(kind: string, env?: Readonly<Record<string, string | undefined>>, now?: Date): EvalDispatch;
44
+ /** Whether this process is in an eval world (any gate named, readable or not): the helpers that keep no record refuse
45
+ * there. */
46
+ export declare function inEvalWorld(env?: Readonly<Record<string, string | undefined>>): boolean;
47
+ /** Whether a dispatch in this world asked past its limit (and was refused). */
48
+ export declare function evalGateOverLimit(dir: string): boolean;
49
+ /** The slots a world's dispatches took, read back by the runner (never following a link). Null when the folder can't
50
+ * be read whole: the runner then counts the world's spending as unknown. */
51
+ export declare function evalGateSlots(dir: string): EvalGateSlot[] | null;
@@ -0,0 +1,199 @@
1
+ /**
2
+ * The eval world's dispatch gate (docs/plans/evals-nightly.md, part C), and the provider-gate logic the runner needs
3
+ * at run time, ported from scripts/provider-gate.mjs so it ships in dist.
4
+ *
5
+ * An eval attempt runs in a world of its own (evals.ts). Before the world starts, the attempt's runner makes a gate
6
+ * folder in the world's state folder (which every agent fence in the world denies) holding `gate.json`: the case's
7
+ * dispatch limit. The world's processes find it through TOOLROLL_EVAL_GATE. Every provider dispatch the world makes
8
+ * takes one numbered slot first (an exclusive create, so two dispatches never take the same one):
9
+ *
10
+ * - every agent run the invocation gateway (invoke.ts) spawns: builder, planner, repair, reviewer, scout;
11
+ * - every lead request, which doesn't go through the gateway (direct API and subscription), and fleet chat's;
12
+ * - the helpers that would call a provider and keep no record (task sizing, flow drafts, subagent turns, the
13
+ * memory pass) refuse in an eval world instead.
14
+ *
15
+ * Past the limit the dispatch refuses and nothing is sent, and the gate remembers it was asked (the attempt then
16
+ * fails). There is no dollar cap (the owner, 2026-10-30): what each dispatch cost is recorded, not limited. Outside an
17
+ * eval world (no TOOLROLL_EVAL_GATE) nothing here changes anything; the variable only ever narrows, and a gate that
18
+ * can't be read refuses every dispatch (fail closed).
19
+ *
20
+ * The runner reads the slots back after the attempt (`evalGateSlots`) and matches them to the world's records; a slot
21
+ * no record accounts for, or a record no slot does, makes the attempt an error with unknown spending.
22
+ */
23
+ import { closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, readdirSync, readSync, writeSync } from "node:fs";
24
+ import { join } from "node:path";
25
+ export const EVAL_GATE_ENV = "TOOLROLL_EVAL_GATE";
26
+ const CONFIG = "gate.json";
27
+ const SLOTS = "slots";
28
+ const OVER = "over-limit";
29
+ const MAX_GATE_BYTES = 4096;
30
+ /** At most `limit` bodies at once; the others start in the order they asked, as each running one ends (provider-gate
31
+ * limiter). */
32
+ export function limiter(limit) {
33
+ if (!Number.isInteger(limit) || limit < 1)
34
+ throw new Error("a limiter takes a whole number, 1 or more");
35
+ let running = 0;
36
+ const queue = [];
37
+ const next = () => { if (running < limit && queue.length > 0) {
38
+ running += 1;
39
+ queue.shift()();
40
+ } };
41
+ return (body) => new Promise(start => { queue.push(start); next(); }).then(async () => {
42
+ try {
43
+ return await body();
44
+ }
45
+ finally {
46
+ running -= 1;
47
+ next();
48
+ }
49
+ });
50
+ }
51
+ /** Read a small regular file without following a link; null when it isn't one, is too big or can't be read. */
52
+ export function readNoFollow(path, maxBytes) {
53
+ let fd;
54
+ try {
55
+ fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | (constants.O_NONBLOCK ?? 0));
56
+ }
57
+ catch {
58
+ return null;
59
+ }
60
+ try {
61
+ const stat = fstatSync(fd);
62
+ if (!stat.isFile() || stat.size > maxBytes)
63
+ return null;
64
+ const out = Buffer.alloc(stat.size);
65
+ let at = 0;
66
+ while (at < out.length) {
67
+ const got = readSync(fd, out, at, out.length - at, at);
68
+ if (got === 0)
69
+ break;
70
+ at += got;
71
+ }
72
+ return at === out.length ? out : null;
73
+ }
74
+ catch {
75
+ return null;
76
+ }
77
+ finally {
78
+ closeSync(fd);
79
+ }
80
+ }
81
+ const validConfig = (value) => {
82
+ if (typeof value !== "object" || value === null)
83
+ return false;
84
+ const one = value;
85
+ return one["v"] === 2 && Number.isInteger(one["max"]) && one["max"] >= 0 && one["max"] <= 50;
86
+ };
87
+ /** Make a world's gate in `dir` (a fresh folder the runner owns): its terms, and an empty slots folder. Refuses a folder
88
+ * that already holds a gate. */
89
+ export function openEvalGate(dir, config) {
90
+ if (!validConfig(config))
91
+ throw new Error("an eval gate needs a whole dispatch limit");
92
+ mkdirSync(join(dir, SLOTS), { recursive: false, mode: 0o700 });
93
+ const fd = openSync(join(dir, CONFIG), constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
94
+ try {
95
+ writeSync(fd, JSON.stringify(config));
96
+ }
97
+ finally {
98
+ closeSync(fd);
99
+ }
100
+ }
101
+ /** The world's gate, as this process's environment names it: null outside an eval world, its terms, or why it can't
102
+ * be read (which refuses every dispatch). */
103
+ export function evalWorldGate(env = process.env) {
104
+ const dir = env[EVAL_GATE_ENV];
105
+ if (dir === undefined || dir === "")
106
+ return null;
107
+ const bytes = readNoFollow(join(dir, CONFIG), MAX_GATE_BYTES);
108
+ if (bytes === null)
109
+ return { dir, problem: "this eval world's dispatch gate can't be read, so nothing is dispatched" };
110
+ let parsed;
111
+ try {
112
+ parsed = JSON.parse(bytes.toString("utf8"));
113
+ }
114
+ catch {
115
+ parsed = null;
116
+ }
117
+ return validConfig(parsed) ? { dir, config: parsed } : { dir, problem: "this eval world's dispatch gate is malformed, so nothing is dispatched" };
118
+ }
119
+ /** Take one dispatch slot of this eval world, before a provider is called. Outside an eval world: `{ active: false }`. */
120
+ export function takeEvalDispatch(kind, env = process.env, now = new Date()) {
121
+ const gate = evalWorldGate(env);
122
+ if (gate === null)
123
+ return { active: false };
124
+ if ("problem" in gate)
125
+ return { active: true, ok: false, why: gate.problem };
126
+ const record = JSON.stringify({ kind: kind.slice(0, 80), pid: process.pid, at: now.toISOString() });
127
+ for (let n = 1; n <= gate.config.max; n++) {
128
+ let fd;
129
+ try {
130
+ fd = openSync(join(gate.dir, SLOTS, `${n}.json`), constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
131
+ }
132
+ catch (error) {
133
+ if (error.code === "EEXIST")
134
+ continue;
135
+ return { active: true, ok: false, why: "this eval world's dispatch gate can't be written, so nothing is dispatched" };
136
+ }
137
+ try {
138
+ writeSync(fd, record);
139
+ }
140
+ finally {
141
+ closeSync(fd);
142
+ }
143
+ return { active: true, ok: true, n };
144
+ }
145
+ // Remembered for the runner: an attempt that asked past its limit is a fail, whatever it built.
146
+ try {
147
+ const fd = openSync(join(gate.dir, OVER), constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW, 0o600);
148
+ closeSync(fd);
149
+ }
150
+ catch { /* the slots alone still hold the limit */ }
151
+ return { active: true, ok: false, why: `this eval case's limit of ${gate.config.max} provider dispatches is reached (eval-dispatch-limit)` };
152
+ }
153
+ /** Whether this process is in an eval world (any gate named, readable or not): the helpers that keep no record refuse
154
+ * there. */
155
+ export function inEvalWorld(env = process.env) {
156
+ return evalWorldGate(env) !== null;
157
+ }
158
+ /** Whether a dispatch in this world asked past its limit (and was refused). */
159
+ export function evalGateOverLimit(dir) {
160
+ try {
161
+ return lstatSync(join(dir, OVER)).isFile();
162
+ }
163
+ catch {
164
+ return false;
165
+ }
166
+ }
167
+ /** The slots a world's dispatches took, read back by the runner (never following a link). Null when the folder can't
168
+ * be read whole: the runner then counts the world's spending as unknown. */
169
+ export function evalGateSlots(dir) {
170
+ let names;
171
+ try {
172
+ names = readdirSync(join(dir, SLOTS));
173
+ }
174
+ catch {
175
+ return null;
176
+ }
177
+ const slots = [];
178
+ for (const name of names) {
179
+ const match = /^([1-9][0-9]{0,2})\.json$/.exec(name);
180
+ if (match === null)
181
+ return null;
182
+ const bytes = readNoFollow(join(dir, SLOTS, name), MAX_GATE_BYTES);
183
+ if (bytes === null)
184
+ return null;
185
+ let parsed;
186
+ try {
187
+ parsed = JSON.parse(bytes.toString("utf8"));
188
+ }
189
+ catch {
190
+ return null;
191
+ }
192
+ if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)
193
+ || typeof parsed["kind"] !== "string" || parsed["kind"] === ""
194
+ || !Number.isSafeInteger(parsed["pid"]) || Number(parsed["pid"]) <= 0 || typeof parsed["at"] !== "string")
195
+ return null;
196
+ slots.push({ n: Number(match[1]), kind: typeof parsed["kind"] === "string" ? parsed["kind"] : "unknown", pid: Number(parsed["pid"]) || 0, at: typeof parsed["at"] === "string" ? parsed["at"] : "" });
197
+ }
198
+ return slots.sort((a, b) => a.n - b.n);
199
+ }
@@ -0,0 +1,39 @@
1
+ /** One required call: in this file, inside this top-level function, a call to this name, the name bound as `from`
2
+ * says (a module specifier its un-renamed named import comes from; `self`: the file's own top-level function; `local`:
3
+ * declared inside `inside`). `args`: the call's arguments, as text, must stay as the base has them. */
4
+ export type EvalHook = {
5
+ file: string;
6
+ inside: string;
7
+ calls: string;
8
+ from: string;
9
+ args?: true;
10
+ why: string;
11
+ };
12
+ /** One function whose body (its text, as the candidate has it) must stay as the base has it: a top-level function, or
13
+ * `Class#method`, a method of a top-level class. */
14
+ export type EvalPinned = {
15
+ file: string;
16
+ fn: string;
17
+ why: string;
18
+ };
19
+ export declare const EVAL_HOOKS: readonly EvalHook[];
20
+ export declare const EVAL_PINNED: readonly EvalPinned[];
21
+ /** The Store methods pinned by body, which no file may write over (storeRewriteProblem, store.ts sealStoreMoneyMethods). */
22
+ export declare const PINNED_STORE_METHODS: readonly string[];
23
+ /**
24
+ * Why a candidate drops a required hook or changes a pinned body, or null. `candidate` gives a listed file's bytes at
25
+ * the candidate (null: missing or unreadable, which needs a person too). With `base` (the plane's own record of where
26
+ * the release started, which the candidate can't rewrite), only what the base holds is required: a project that isn't
27
+ * Toolroll holds none, and a hook not yet landed can't be dropped.
28
+ */
29
+ export declare function evalHookProblem(candidate: (file: string) => Buffer | null, base?: (file: string) => Buffer | null, hooks?: readonly EvalHook[], pinned?: readonly EvalPinned[]): string | null;
30
+ /** Every file the manifest reads, for one verified read of each tree. */
31
+ export declare const EVAL_HOOK_FILES: readonly string[];
32
+ /**
33
+ * Why a changed file could write over a pinned Store method, or null (PR #263 security scan). Each changed JavaScript
34
+ * or TypeScript file, store.ts included (its class's own method declarations exempt; their bodies are pinned), is held to
35
+ * storeRewrites: a construct the base's copy of that file didn't already have (counted, by the sha256 of its whole
36
+ * text) needs a person. A tree or file that can't be read, or a file that doesn't parse, needs one too; `paths` are the
37
+ * files the verified diff says exist at the candidate, so no bytes is a failed read, never a deletion.
38
+ */
39
+ export declare function storeRewriteProblem(paths: readonly string[], candidate: ((file: string) => Buffer | null) | null, base: ((file: string) => Buffer | null) | null, pinned?: readonly string[]): string | null;