toolroll 0.9.79 → 0.9.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/dist/cli.js +1 -0
- package/dist/commit-extract.d.ts +18 -0
- package/dist/commit-extract.js +39 -1
- package/dist/converse.d.ts +12 -0
- package/dist/converse.js +18 -2
- package/dist/deploy-gate.js +8 -0
- package/dist/eval-gate.d.ts +51 -0
- package/dist/eval-gate.js +199 -0
- package/dist/eval-hooks.d.ts +39 -0
- package/dist/eval-hooks.js +477 -0
- package/dist/eval-oracle-main.d.ts +1 -0
- package/dist/eval-oracle-main.js +31 -0
- package/dist/eval-oracle.d.ts +60 -0
- package/dist/eval-oracle.js +217 -0
- package/dist/eval-world.d.ts +17 -0
- package/dist/eval-world.js +97 -0
- package/dist/evals.d.ts +299 -0
- package/dist/evals.js +964 -0
- package/dist/exec.js +15 -1
- package/dist/flow-draft.js +4 -0
- package/dist/flow-sort.js +4 -0
- package/dist/github-jobs.d.ts +25 -0
- package/dist/github-jobs.js +22 -0
- package/dist/guides.js +12 -9
- package/dist/invoke.d.ts +1 -1
- package/dist/invoke.js +14 -0
- package/dist/lead.js +4 -1
- package/dist/memory-pass.js +4 -0
- package/dist/nightly-check.d.ts +4 -1
- package/dist/nightly-check.js +5 -2
- package/dist/nightly-github.d.ts +316 -0
- package/dist/nightly-github.js +1022 -0
- package/dist/operate.d.ts +91 -1
- package/dist/operate.js +158 -1
- package/dist/project-knowledge.d.ts +4 -4
- package/dist/provider.d.ts +3 -0
- package/dist/provider.js +9 -2
- package/dist/release-adapters.d.ts +21 -21
- package/dist/release-adapters.js +53 -20
- package/dist/release-auto.d.ts +1 -1
- package/dist/release-auto.js +26 -0
- package/dist/release-card.js +3 -0
- package/dist/release-cli.js +59 -3
- package/dist/release-view.d.ts +3 -0
- package/dist/release-view.js +2 -0
- package/dist/release.d.ts +36 -17
- package/dist/release.js +74 -24
- package/dist/reported-usage.d.ts +69 -0
- package/dist/reported-usage.js +97 -0
- package/dist/server/chat.js +4 -0
- package/dist/server/render-tasks.js +1 -0
- package/dist/server/tasks.js +8 -1
- package/dist/store.d.ts +109 -1
- package/dist/store.js +394 -4
- package/dist/subagents.js +4 -0
- package/dist/subscription-chat.d.ts +2 -0
- package/dist/subscription-chat.js +52 -13
- package/dist/task-sizing.js +4 -0
- package/dist/teardown-git.d.ts +9 -2
- package/dist/teardown-git.js +56 -33
- package/evals/cases/catalog/case.json +25 -0
- package/evals/cases/catalog/fixture/README.md +3 -0
- package/evals/cases/catalog/fixture/package.json +8 -0
- package/evals/cases/catalog/fixture/test/catalog.test.mjs +4 -0
- package/evals/cases/catalog/oracle.mjs +23 -0
- package/evals/cases/catalog/probe.mjs +10 -0
- package/evals/cases/catalog/source.md +9 -0
- package/evals/cases/csv-quoting/case.json +23 -0
- package/evals/cases/csv-quoting/fixture/README.md +3 -0
- package/evals/cases/csv-quoting/fixture/csv.js +1 -0
- package/evals/cases/csv-quoting/fixture/package.json +8 -0
- package/evals/cases/csv-quoting/fixture/test/csv.test.mjs +5 -0
- package/evals/cases/csv-quoting/oracle.mjs +14 -0
- package/evals/cases/csv-quoting/probe.mjs +9 -0
- package/evals/cases/csv-quoting/source.md +9 -0
- package/evals/cases/dedupe/case.json +23 -0
- package/evals/cases/dedupe/fixture/README.md +3 -0
- package/evals/cases/dedupe/fixture/dedupe.js +1 -0
- package/evals/cases/dedupe/fixture/package.json +8 -0
- package/evals/cases/dedupe/fixture/test/dedupe.test.mjs +5 -0
- package/evals/cases/dedupe/oracle.mjs +16 -0
- package/evals/cases/dedupe/probe.mjs +13 -0
- package/evals/cases/dedupe/source.md +9 -0
- package/evals/cases/endpoint/case.json +25 -0
- package/evals/cases/endpoint/fixture/README.md +3 -0
- package/evals/cases/endpoint/fixture/package.json +8 -0
- package/evals/cases/endpoint/fixture/test/endpoint.test.mjs +4 -0
- package/evals/cases/endpoint/oracle.mjs +27 -0
- package/evals/cases/endpoint/probe.mjs +10 -0
- package/evals/cases/endpoint/source.md +11 -0
- package/evals/cases/math-divide/case.json +23 -0
- package/evals/cases/math-divide/fixture/README.md +3 -0
- package/evals/cases/math-divide/fixture/package.json +8 -0
- package/evals/cases/math-divide/fixture/src/math.js +1 -0
- package/evals/cases/math-divide/fixture/test/math.test.js +5 -0
- package/evals/cases/math-divide/oracle.mjs +13 -0
- package/evals/cases/math-divide/probe.mjs +9 -0
- package/evals/cases/math-divide/source.md +9 -0
- package/evals/cases/math-multiply/case.json +23 -0
- package/evals/cases/math-multiply/fixture/README.md +3 -0
- package/evals/cases/math-multiply/fixture/package.json +8 -0
- package/evals/cases/math-multiply/fixture/src/math.js +2 -0
- package/evals/cases/math-multiply/fixture/test/math.test.js +5 -0
- package/evals/cases/math-multiply/oracle.mjs +12 -0
- package/evals/cases/math-multiply/probe.mjs +9 -0
- package/evals/cases/math-multiply/source.md +9 -0
- package/evals/cases/math-subtract/case.json +23 -0
- package/evals/cases/math-subtract/fixture/README.md +3 -0
- package/evals/cases/math-subtract/fixture/package.json +8 -0
- package/evals/cases/math-subtract/fixture/src/math.js +1 -0
- package/evals/cases/math-subtract/fixture/test/math.test.js +5 -0
- package/evals/cases/math-subtract/oracle.mjs +12 -0
- package/evals/cases/math-subtract/probe.mjs +9 -0
- package/evals/cases/math-subtract/source.md +9 -0
- package/evals/cases/merge-config/case.json +23 -0
- package/evals/cases/merge-config/fixture/README.md +3 -0
- package/evals/cases/merge-config/fixture/config.js +1 -0
- package/evals/cases/merge-config/fixture/package.json +8 -0
- package/evals/cases/merge-config/fixture/test/config.test.mjs +5 -0
- package/evals/cases/merge-config/oracle.mjs +28 -0
- package/evals/cases/merge-config/probe.mjs +15 -0
- package/evals/cases/merge-config/source.md +9 -0
- package/evals/cases/paginate/case.json +23 -0
- package/evals/cases/paginate/fixture/README.md +3 -0
- package/evals/cases/paginate/fixture/package.json +8 -0
- package/evals/cases/paginate/fixture/paginate.js +1 -0
- package/evals/cases/paginate/fixture/test/paginate.test.mjs +5 -0
- package/evals/cases/paginate/oracle.mjs +18 -0
- package/evals/cases/paginate/probe.mjs +13 -0
- package/evals/cases/paginate/source.md +9 -0
- package/evals/cases/round2-asks/case.json +24 -0
- package/evals/cases/round2-asks/fixture/README.md +3 -0
- package/evals/cases/round2-asks/fixture/package.json +8 -0
- package/evals/cases/round2-asks/fixture/src/math.js +1 -0
- package/evals/cases/round2-asks/fixture/test/math.test.js +5 -0
- package/evals/cases/round2-asks/source.md +11 -0
- package/evals/manifest.json +15 -0
- package/package.json +2 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,52 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.9.82 — 2026-10-31
|
|
4
|
+
|
|
5
|
+
- **A failing nightly holds your releases.** Turn it on with
|
|
6
|
+
`toolroll release nightly --github` (your password). When a lane of your
|
|
7
|
+
project's GitHub nightly fails on main, every deploy, the merge and the tag
|
|
8
|
+
of a release wait until a later nightly passes that lane, and Needs you
|
|
9
|
+
says which lane and why. A release whose own pull request re-ran that lane
|
|
10
|
+
after the failure, with the same CI files, goes through. A nightly that
|
|
11
|
+
can't be read gives a warning, never a pass.
|
|
12
|
+
- **Every run and chat turn records the model and cost the provider
|
|
13
|
+
reported:** the exact model id, provider turns, cache-aware tokens, and
|
|
14
|
+
cost, with API dollars, subscription usage and unknown kept apart (unknown
|
|
15
|
+
is never shown as zero).
|
|
16
|
+
- **Evals (groundwork).** `toolroll evals` runs repeatable test tasks in
|
|
17
|
+
throwaway worlds, graded by a separate checker process the agent can't
|
|
18
|
+
reach. Ten starter tasks ship as drafts; real comparison runs stay off
|
|
19
|
+
until they're validated.
|
|
20
|
+
- **CI:** required checks report under their exact names on every pull
|
|
21
|
+
request; a slow git config read now says it timed out instead of blaming
|
|
22
|
+
the project's config.
|
|
23
|
+
|
|
24
|
+
## 0.9.81 — 2026-10-30
|
|
25
|
+
|
|
26
|
+
- **The release proof works after the merge.** GitHub empties a CI run's
|
|
27
|
+
pull-request link once the pull request merges and its branch is deleted,
|
|
28
|
+
so 0.9.80's tag could never prove its checks and wasn't published. A run
|
|
29
|
+
with no link now stands only when its branch has had exactly one pull
|
|
30
|
+
request, ever, and that pull request merged into main. 0.9.80's changes
|
|
31
|
+
ship in this release.
|
|
32
|
+
|
|
33
|
+
## 0.9.80 — 2026-10-30
|
|
34
|
+
|
|
35
|
+
- **Each test runs once, where it adds something** (docs/plans/test-matrix.md).
|
|
36
|
+
Pull requests run the unit suite on Node 24 (Linux and macOS) and every
|
|
37
|
+
area lane (Windows, Linux fence and cgroup, git 2.41 and 2.47); a pull
|
|
38
|
+
request that only changes Markdown under `docs/`, the changelog or the
|
|
39
|
+
README skips the area lanes. Node 22 runs on Linux and macOS once per
|
|
40
|
+
release, on the `release-*` branch's pull request, in the new
|
|
41
|
+
Release candidate workflow. Release tags no longer rerun the macOS suites:
|
|
42
|
+
a release proof reads GitHub's record that the tagged tree passed every
|
|
43
|
+
required check on its pull request before npm publication. A push to main
|
|
44
|
+
typechecks and builds, and runs the full CI only when it can't prove the
|
|
45
|
+
merge commit was tested. The nightly runs everything. A release now needs
|
|
46
|
+
every required check executed, never skipped, and refuses to merge while
|
|
47
|
+
main's branch protection doesn't require all of them. Hosted unit shards
|
|
48
|
+
require their platform's native fence attacks to run.
|
|
49
|
+
|
|
3
50
|
## 0.9.79 — 2026-10-29
|
|
4
51
|
|
|
5
52
|
- **`toolroll release` rides out a short network failure instead of
|
package/dist/cli.js
CHANGED
|
@@ -108,6 +108,7 @@ export function parseArgs(argv) {
|
|
|
108
108
|
/** The commands that operate the queue rather than report on the world. */
|
|
109
109
|
export const OPERATE_COMMANDS = new Set([
|
|
110
110
|
"up",
|
|
111
|
+
"evals",
|
|
111
112
|
// Only `lead token` reaches here; the central service's lead verbs answer first.
|
|
112
113
|
"lead",
|
|
113
114
|
"onboard",
|
package/dist/commit-extract.d.ts
CHANGED
|
@@ -71,6 +71,11 @@ export type Verified<T> = {
|
|
|
71
71
|
export type GitRunner = {
|
|
72
72
|
git?: string;
|
|
73
73
|
env?: NodeJS.ProcessEnv;
|
|
74
|
+
/** Each git call's time limit (default five minutes). */
|
|
75
|
+
timeoutMs?: number;
|
|
76
|
+
/** An absolute wall-clock deadline (ms): every git subprocess, setup included, gets only what is left of it, and a
|
|
77
|
+
* read past it fails (teardown-git.ts viewedGitSync). */
|
|
78
|
+
deadline?: number;
|
|
74
79
|
};
|
|
75
80
|
/** One commit, hashed here and parsed from its own bytes. */
|
|
76
81
|
export declare function readVerifiedCommit(gitDir: string, sha: string): Verified<VerifiedCommit>;
|
|
@@ -114,4 +119,17 @@ export declare function verifiedRange(gitDir: string, from: string, to: string,
|
|
|
114
119
|
max?: number;
|
|
115
120
|
reach?: number;
|
|
116
121
|
}): Verified<VerifiedCommit[]>;
|
|
122
|
+
/**
|
|
123
|
+
* Whether `until` lies on `from`'s first-parent history, within a deadline (nightly-github.ts): "yes" when the walk
|
|
124
|
+
* reaches it; "no" only when it is PROVEN not to — `until` is itself a readable, verified commit and every commit from
|
|
125
|
+
* `from` back to the root was read and verified without meeting it; "unknown" for anything less certain (an object
|
|
126
|
+
* missing or unreadable, `max` commits walked, the deadline reached). Every git call is bounded by what is left of
|
|
127
|
+
* the deadline, at most `callMs`.
|
|
128
|
+
*/
|
|
129
|
+
export declare function verifiedFirstParentAncestry(gitDir: string, from: string, until: string, options: {
|
|
130
|
+
deadline: number;
|
|
131
|
+
now?: () => number;
|
|
132
|
+
max?: number;
|
|
133
|
+
callMs?: number;
|
|
134
|
+
}): "yes" | "no" | "unknown";
|
|
117
135
|
export {};
|
package/dist/commit-extract.js
CHANGED
|
@@ -212,7 +212,7 @@ function protectedGit(gitDir, runner, extra) {
|
|
|
212
212
|
if (checkout === null)
|
|
213
213
|
return { status: null, stdout: Buffer.alloc(0), refused: "it isn't a checkout's .git folder, which this git can only read through a view" };
|
|
214
214
|
try {
|
|
215
|
-
const ran = viewedGitSync(checkout, [...base, ...args], { buffer: true, timeoutMs: 300_000, maxBuffer: MAX_BYTES, ...(input === undefined ? {} : { input }), ...(runner.git === undefined ? {} : { git: runner.git }) });
|
|
215
|
+
const ran = viewedGitSync(checkout, [...base, ...args], { buffer: true, timeoutMs: runner.timeoutMs ?? 300_000, ...(runner.deadline === undefined ? {} : { deadline: runner.deadline }), maxBuffer: MAX_BYTES, ...(input === undefined ? {} : { input }), ...(runner.git === undefined ? {} : { git: runner.git }) });
|
|
216
216
|
return { status: ran.status, stdout: ran.stdout, refused: null };
|
|
217
217
|
}
|
|
218
218
|
catch (error) {
|
|
@@ -567,3 +567,41 @@ export function verifiedRange(gitDir, from, to, options = {}) {
|
|
|
567
567
|
const unreached = wanted.find(one => !reached.has(one));
|
|
568
568
|
return unreached === undefined ? { ok: true, value: commits } : { ok: false, message: `${unreached.slice(0, 7)} isn't verified as part of the last release's history.` };
|
|
569
569
|
}
|
|
570
|
+
/**
|
|
571
|
+
* Whether `until` lies on `from`'s first-parent history, within a deadline (nightly-github.ts): "yes" when the walk
|
|
572
|
+
* reaches it; "no" only when it is PROVEN not to — `until` is itself a readable, verified commit and every commit from
|
|
573
|
+
* `from` back to the root was read and verified without meeting it; "unknown" for anything less certain (an object
|
|
574
|
+
* missing or unreadable, `max` commits walked, the deadline reached). Every git call is bounded by what is left of
|
|
575
|
+
* the deadline, at most `callMs`.
|
|
576
|
+
*/
|
|
577
|
+
export function verifiedFirstParentAncestry(gitDir, from, until, options) {
|
|
578
|
+
const now = options.now ?? Date.now;
|
|
579
|
+
const left = () => options.deadline - now();
|
|
580
|
+
// Every answer is checked against the deadline as it is given: nothing past it is ever "yes" or "no".
|
|
581
|
+
const answer = (value) => left() > 0 ? value : "unknown";
|
|
582
|
+
if (left() <= 0)
|
|
583
|
+
return "unknown";
|
|
584
|
+
// The absolute deadline goes down to every subprocess (setup included); each call is also held to `callMs`.
|
|
585
|
+
const reader = verifiedReader(gitDir, { timeoutMs: options.callMs ?? 10_000, deadline: options.deadline });
|
|
586
|
+
if (!reader.ok)
|
|
587
|
+
return "unknown";
|
|
588
|
+
if (from === until)
|
|
589
|
+
return readCommitWith(reader.value, until).ok ? answer("yes") : "unknown";
|
|
590
|
+
if (left() <= 0 || !readCommitWith(reader.value, until).ok)
|
|
591
|
+
return "unknown";
|
|
592
|
+
let at = from;
|
|
593
|
+
for (let walked = 0; walked < (options.max ?? 20_000); walked++) {
|
|
594
|
+
if (left() <= 0)
|
|
595
|
+
return "unknown";
|
|
596
|
+
const commit = readCommitWith(reader.value, at);
|
|
597
|
+
if (!commit.ok)
|
|
598
|
+
return "unknown";
|
|
599
|
+
const parent = commit.value.parents[0];
|
|
600
|
+
if (parent === undefined)
|
|
601
|
+
return answer("no");
|
|
602
|
+
if (parent === until)
|
|
603
|
+
return answer("yes");
|
|
604
|
+
at = parent;
|
|
605
|
+
}
|
|
606
|
+
return "unknown";
|
|
607
|
+
}
|
package/dist/converse.d.ts
CHANGED
|
@@ -132,6 +132,9 @@ export type ProviderAnswer = {
|
|
|
132
132
|
/** OpenRouter reports the actual charge; settlement takes the HIGHER of
|
|
133
133
|
* this and the pinned math — the ledger never undercounts. */
|
|
134
134
|
reportedCostMicrousd: number | null;
|
|
135
|
+
/** v140: the exact model the response body names, and the cost it reported in dollars (absent or null: unreported). */
|
|
136
|
+
modelReported?: string | null;
|
|
137
|
+
reportedCostUsd?: number | null;
|
|
135
138
|
};
|
|
136
139
|
/** Strict parse of the PROVIDER response body (layer one). */
|
|
137
140
|
/** A token count is a non-negative safe integer under a billion (mate arc,
|
|
@@ -184,6 +187,15 @@ export type LeadProviderAnswer = {
|
|
|
184
187
|
reportedCostMicrousd: number | null;
|
|
185
188
|
cacheWriteTokens?: number;
|
|
186
189
|
cacheReadTokens?: number;
|
|
190
|
+
/** v140 (docs/plans/evals-nightly.md B1), each from the provider's own report and absent or null when it gave none
|
|
191
|
+
* (never 0): the exact model it says answered, its own turn count (`num_turns`), and the cost it reported in dollars.
|
|
192
|
+
* `reportedCostMicrousd` stays API money only; a subscription harness's reported cost is `reportedCostUsd` alone. */
|
|
193
|
+
modelReported?: string | null;
|
|
194
|
+
providerTurns?: number | null;
|
|
195
|
+
reportedCostUsd?: number | null;
|
|
196
|
+
/** v141: how a subscription harness's own events said this request was billed (its init's key source); null or
|
|
197
|
+
* absent when they didn't say (unknown money, never assumed to be the plan). */
|
|
198
|
+
billingReported?: "subscription" | "api-key" | null;
|
|
187
199
|
};
|
|
188
200
|
/** Anthropic prices a cache write at 1.25x and a cache read at 0.1x the input price (5-minute cache). */
|
|
189
201
|
export declare const CACHE_WRITE_RATIO = 1.25;
|
package/dist/converse.js
CHANGED
|
@@ -19,10 +19,12 @@
|
|
|
19
19
|
* micro-dollars per token, rounded up. A model without a pinned price
|
|
20
20
|
* cannot be spoken to — the reservation math would be fiction.
|
|
21
21
|
*/
|
|
22
|
+
import { takeEvalDispatch } from "./eval-gate.js";
|
|
22
23
|
import { TASK_TEXT_LIMITS } from "./task-text.js";
|
|
23
24
|
import { createHash } from "node:crypto";
|
|
24
25
|
import { hasForbiddenControls, hasDisguisedText } from "./decision.js";
|
|
25
26
|
import { parseAcceptanceCriteria } from "./scope.js";
|
|
27
|
+
import { reportedCostUsdOf, reportedModelOf } from "./reported-usage.js";
|
|
26
28
|
// ---------------------------------------------------------------- limits
|
|
27
29
|
export const WRAPPER_CAP_BYTES = 262_144; // the provider response body
|
|
28
30
|
export const ENVELOPE_CAP_BYTES = 65_536; // the assistant text inside it
|
|
@@ -481,6 +483,7 @@ export function parseLeadProviderWrapper(provider, bytes) {
|
|
|
481
483
|
if (write === null || read === null)
|
|
482
484
|
return { ok: false, problem: "no-usage" };
|
|
483
485
|
return { ok: true, answer: { text, calls, tokensIn: usage.tokensIn + (write ?? 0) + (read ?? 0), tokensOut: usage.tokensOut, reportedCostMicrousd: null,
|
|
486
|
+
modelReported: reportedModelOf(body["model"]), providerTurns: null, reportedCostUsd: null,
|
|
484
487
|
...(write === undefined ? {} : { cacheWriteTokens: write }), ...(read === undefined ? {} : { cacheReadTokens: read }) } };
|
|
485
488
|
}
|
|
486
489
|
const choices = body["choices"];
|
|
@@ -527,7 +530,8 @@ export function parseLeadProviderWrapper(provider, bytes) {
|
|
|
527
530
|
return { ok: false, problem: "bad-cost" };
|
|
528
531
|
reportedCostMicrousd = micro;
|
|
529
532
|
}
|
|
530
|
-
return { ok: true, answer: { text, calls, tokensIn: usage.tokensIn, tokensOut: usage.tokensOut, reportedCostMicrousd
|
|
533
|
+
return { ok: true, answer: { text, calls, tokensIn: usage.tokensIn, tokensOut: usage.tokensOut, reportedCostMicrousd,
|
|
534
|
+
modelReported: reportedModelOf(body["model"]), providerTurns: null, reportedCostUsd: reportedCostUsdOf(rawCost) } };
|
|
531
535
|
}
|
|
532
536
|
/** The lead's request: system contract, the data document as the first
|
|
533
537
|
* operator message, the history in provider-native shape, the tools. */
|
|
@@ -593,6 +597,10 @@ export function composeLeadRequest(args) {
|
|
|
593
597
|
}
|
|
594
598
|
/** The lead's network call: the same transport posture as fleet chat's, the tool-capable parser at the end. */
|
|
595
599
|
export async function performLeadRequest(request, provider, signal, fetcher = fetch) {
|
|
600
|
+
// An eval world (eval-gate.ts): each request takes one of the attempt's dispatch slots. Past the limit nothing is sent.
|
|
601
|
+
const evalSlot = takeEvalDispatch(`lead:${provider}`);
|
|
602
|
+
if (evalSlot.active && !evalSlot.ok)
|
|
603
|
+
return { ok: false, problem: "status-eval-dispatch-limit" };
|
|
596
604
|
let response;
|
|
597
605
|
try {
|
|
598
606
|
response = await fetcher(request.url, { method: "POST", headers: request.headers, body: request.body, redirect: "error", signal });
|
|
@@ -634,7 +642,7 @@ export function parseProviderWrapper(provider, bytes) {
|
|
|
634
642
|
const tokensOut = usage === undefined ? NaN : Number(usage["output_tokens"]);
|
|
635
643
|
if (!tokenCount(tokensIn) || !tokenCount(tokensOut))
|
|
636
644
|
return { ok: false, problem: "no-usage" };
|
|
637
|
-
return { ok: true, answer: { text: block["text"], tokensIn, tokensOut, reportedCostMicrousd: null } };
|
|
645
|
+
return { ok: true, answer: { text: block["text"], tokensIn, tokensOut, reportedCostMicrousd: null, modelReported: reportedModelOf(body["model"]), reportedCostUsd: null } };
|
|
638
646
|
}
|
|
639
647
|
// openrouter-api (OpenAI-compatible)
|
|
640
648
|
const choices = body["choices"];
|
|
@@ -660,6 +668,9 @@ export function parseProviderWrapper(provider, bytes) {
|
|
|
660
668
|
tokensIn,
|
|
661
669
|
tokensOut,
|
|
662
670
|
reportedCostMicrousd: Number.isFinite(reportedCost) && reportedCost >= 0 ? Math.ceil(reportedCost * 1_000_000) : null,
|
|
671
|
+
modelReported: reportedModelOf(body["model"]),
|
|
672
|
+
// The body's own number only: Number(null) would read an unreported cost as 0.
|
|
673
|
+
reportedCostUsd: usage === undefined ? null : reportedCostUsdOf(usage["cost"]),
|
|
663
674
|
},
|
|
664
675
|
};
|
|
665
676
|
}
|
|
@@ -801,6 +812,11 @@ export async function readCappedBody(response, cap) {
|
|
|
801
812
|
* wall clock, streamed cap, wrapper strictness.
|
|
802
813
|
*/
|
|
803
814
|
export async function performChatRequest(args, fetcher = fetch) {
|
|
815
|
+
// An eval world (eval-gate.ts, Codex round 1 #3): fleet chat's request takes a dispatch slot too; past the limit, or
|
|
816
|
+
// with a gate that can't be read, nothing is sent.
|
|
817
|
+
const evalSlot = takeEvalDispatch(`chat:${args.provider}`);
|
|
818
|
+
if (evalSlot.active && !evalSlot.ok)
|
|
819
|
+
return { ok: false, problem: "status-eval-dispatch-limit" };
|
|
804
820
|
const request = composeRequest(args);
|
|
805
821
|
let response;
|
|
806
822
|
try {
|
package/dist/deploy-gate.js
CHANGED
|
@@ -9,6 +9,7 @@ import { assignmentOf } from "./assignment.js";
|
|
|
9
9
|
import { liveReleaseOf, personApprovalOf, releaseApprovalOf } from "./scope.js";
|
|
10
10
|
import { verificationEvidence } from "./verification-evidence.js";
|
|
11
11
|
import { releaseCiRunningOf } from "./release-ci-hold.js";
|
|
12
|
+
import { nightlyDebtHold } from "./nightly-github.js";
|
|
12
13
|
/** Whether run `runId` is a deployable candidate now, read only from the plane's records and this runtime's code. */
|
|
13
14
|
export function deploymentGate(store, input) {
|
|
14
15
|
const refuse = (message) => ({ ok: false, message });
|
|
@@ -40,6 +41,13 @@ export function deploymentGate(store, input) {
|
|
|
40
41
|
return refuse(`This release can't deploy: ${release.why}.`);
|
|
41
42
|
const anchorPerson = release.applies && release.ok
|
|
42
43
|
? (input.releaseFlow === true ? releaseApprovalOf(store, release.authority.anchorTask, input.now) : personApprovalOf(store, release.authority.anchorTask)) : null;
|
|
44
|
+
// GitHub's nightly lane debt (nightly-github.ts) holds every deploy of the project, a person's approval or not,
|
|
45
|
+
// unless this exact candidate re-ran that lane after the failure and passed (an exemption read at the boundary).
|
|
46
|
+
if (ref.repo !== null) {
|
|
47
|
+
const nightly = nightlyDebtHold(store, { repo: ref.repo, taskId: ref.externalId, candidate: scope?.candidate ?? head, phase: "deploy", release: release.applies, now: input.now });
|
|
48
|
+
if (nightly !== null)
|
|
49
|
+
return refuse(nightly.why);
|
|
50
|
+
}
|
|
43
51
|
const person = anchorPerson ?? personApprovalOf(store, ref.externalId);
|
|
44
52
|
if (!person.ok)
|
|
45
53
|
return refuse(`A person has to approve this candidate's scope: ${person.why}.`);
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
export declare const EVAL_GATE_ENV = "TOOLROLL_EVAL_GATE";
|
|
2
|
+
/** The gate's terms: how many provider dispatches the attempt may make. */
|
|
3
|
+
export type EvalGateConfig = {
|
|
4
|
+
v: 2;
|
|
5
|
+
max: number;
|
|
6
|
+
};
|
|
7
|
+
/** One slot a dispatch took. */
|
|
8
|
+
export type EvalGateSlot = {
|
|
9
|
+
n: number;
|
|
10
|
+
kind: string;
|
|
11
|
+
pid: number;
|
|
12
|
+
at: string;
|
|
13
|
+
};
|
|
14
|
+
export type EvalDispatch = {
|
|
15
|
+
active: false;
|
|
16
|
+
} | {
|
|
17
|
+
active: true;
|
|
18
|
+
ok: true;
|
|
19
|
+
n: number;
|
|
20
|
+
} | {
|
|
21
|
+
active: true;
|
|
22
|
+
ok: false;
|
|
23
|
+
why: string;
|
|
24
|
+
};
|
|
25
|
+
/** At most `limit` bodies at once; the others start in the order they asked, as each running one ends (provider-gate
|
|
26
|
+
* limiter). */
|
|
27
|
+
export declare function limiter(limit: number): <T>(body: () => Promise<T>) => Promise<T>;
|
|
28
|
+
/** Read a small regular file without following a link; null when it isn't one, is too big or can't be read. */
|
|
29
|
+
export declare function readNoFollow(path: string, maxBytes: number): Buffer | null;
|
|
30
|
+
/** Make a world's gate in `dir` (a fresh folder the runner owns): its terms, and an empty slots folder. Refuses a folder
|
|
31
|
+
* that already holds a gate. */
|
|
32
|
+
export declare function openEvalGate(dir: string, config: EvalGateConfig): void;
|
|
33
|
+
/** The world's gate, as this process's environment names it: null outside an eval world, its terms, or why it can't
|
|
34
|
+
* be read (which refuses every dispatch). */
|
|
35
|
+
export declare function evalWorldGate(env?: Readonly<Record<string, string | undefined>>): {
|
|
36
|
+
dir: string;
|
|
37
|
+
config: EvalGateConfig;
|
|
38
|
+
} | {
|
|
39
|
+
dir: string;
|
|
40
|
+
problem: string;
|
|
41
|
+
} | null;
|
|
42
|
+
/** Take one dispatch slot of this eval world, before a provider is called. Outside an eval world: `{ active: false }`. */
|
|
43
|
+
export declare function takeEvalDispatch(kind: string, env?: Readonly<Record<string, string | undefined>>, now?: Date): EvalDispatch;
|
|
44
|
+
/** Whether this process is in an eval world (any gate named, readable or not): the helpers that keep no record refuse
|
|
45
|
+
* there. */
|
|
46
|
+
export declare function inEvalWorld(env?: Readonly<Record<string, string | undefined>>): boolean;
|
|
47
|
+
/** Whether a dispatch in this world asked past its limit (and was refused). */
|
|
48
|
+
export declare function evalGateOverLimit(dir: string): boolean;
|
|
49
|
+
/** The slots a world's dispatches took, read back by the runner (never following a link). Null when the folder can't
|
|
50
|
+
* be read whole: the runner then counts the world's spending as unknown. */
|
|
51
|
+
export declare function evalGateSlots(dir: string): EvalGateSlot[] | null;
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The eval world's dispatch gate (docs/plans/evals-nightly.md, part C), and the provider-gate logic the runner needs
|
|
3
|
+
* at run time, ported from scripts/provider-gate.mjs so it ships in dist.
|
|
4
|
+
*
|
|
5
|
+
* An eval attempt runs in a world of its own (evals.ts). Before the world starts, the attempt's runner makes a gate
|
|
6
|
+
* folder in the world's state folder (which every agent fence in the world denies) holding `gate.json`: the case's
|
|
7
|
+
* dispatch limit. The world's processes find it through TOOLROLL_EVAL_GATE. Every provider dispatch the world makes
|
|
8
|
+
* takes one numbered slot first (an exclusive create, so two dispatches never take the same one):
|
|
9
|
+
*
|
|
10
|
+
* - every agent run the invocation gateway (invoke.ts) spawns: builder, planner, repair, reviewer, scout;
|
|
11
|
+
* - every lead request, which doesn't go through the gateway (direct API and subscription), and fleet chat's;
|
|
12
|
+
* - the helpers that would call a provider and keep no record (task sizing, flow drafts, subagent turns, the
|
|
13
|
+
* memory pass) refuse in an eval world instead.
|
|
14
|
+
*
|
|
15
|
+
* Past the limit the dispatch refuses and nothing is sent, and the gate remembers it was asked (the attempt then
|
|
16
|
+
* fails). There is no dollar cap (the owner, 2026-10-30): what each dispatch cost is recorded, not limited. Outside an
|
|
17
|
+
* eval world (no TOOLROLL_EVAL_GATE) nothing here changes anything; the variable only ever narrows, and a gate that
|
|
18
|
+
* can't be read refuses every dispatch (fail closed).
|
|
19
|
+
*
|
|
20
|
+
* The runner reads the slots back after the attempt (`evalGateSlots`) and matches them to the world's records; a slot
|
|
21
|
+
* no record accounts for, or a record no slot does, makes the attempt an error with unknown spending.
|
|
22
|
+
*/
|
|
23
|
+
import { closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, readdirSync, readSync, writeSync } from "node:fs";
|
|
24
|
+
import { join } from "node:path";
|
|
25
|
+
export const EVAL_GATE_ENV = "TOOLROLL_EVAL_GATE";
|
|
26
|
+
const CONFIG = "gate.json";
|
|
27
|
+
const SLOTS = "slots";
|
|
28
|
+
const OVER = "over-limit";
|
|
29
|
+
const MAX_GATE_BYTES = 4096;
|
|
30
|
+
/** At most `limit` bodies at once; the others start in the order they asked, as each running one ends (provider-gate
|
|
31
|
+
* limiter). */
|
|
32
|
+
export function limiter(limit) {
|
|
33
|
+
if (!Number.isInteger(limit) || limit < 1)
|
|
34
|
+
throw new Error("a limiter takes a whole number, 1 or more");
|
|
35
|
+
let running = 0;
|
|
36
|
+
const queue = [];
|
|
37
|
+
const next = () => { if (running < limit && queue.length > 0) {
|
|
38
|
+
running += 1;
|
|
39
|
+
queue.shift()();
|
|
40
|
+
} };
|
|
41
|
+
return (body) => new Promise(start => { queue.push(start); next(); }).then(async () => {
|
|
42
|
+
try {
|
|
43
|
+
return await body();
|
|
44
|
+
}
|
|
45
|
+
finally {
|
|
46
|
+
running -= 1;
|
|
47
|
+
next();
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
/** Read a small regular file without following a link; null when it isn't one, is too big or can't be read. */
|
|
52
|
+
export function readNoFollow(path, maxBytes) {
|
|
53
|
+
let fd;
|
|
54
|
+
try {
|
|
55
|
+
fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | (constants.O_NONBLOCK ?? 0));
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
try {
|
|
61
|
+
const stat = fstatSync(fd);
|
|
62
|
+
if (!stat.isFile() || stat.size > maxBytes)
|
|
63
|
+
return null;
|
|
64
|
+
const out = Buffer.alloc(stat.size);
|
|
65
|
+
let at = 0;
|
|
66
|
+
while (at < out.length) {
|
|
67
|
+
const got = readSync(fd, out, at, out.length - at, at);
|
|
68
|
+
if (got === 0)
|
|
69
|
+
break;
|
|
70
|
+
at += got;
|
|
71
|
+
}
|
|
72
|
+
return at === out.length ? out : null;
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
finally {
|
|
78
|
+
closeSync(fd);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
const validConfig = (value) => {
|
|
82
|
+
if (typeof value !== "object" || value === null)
|
|
83
|
+
return false;
|
|
84
|
+
const one = value;
|
|
85
|
+
return one["v"] === 2 && Number.isInteger(one["max"]) && one["max"] >= 0 && one["max"] <= 50;
|
|
86
|
+
};
|
|
87
|
+
/** Make a world's gate in `dir` (a fresh folder the runner owns): its terms, and an empty slots folder. Refuses a folder
|
|
88
|
+
* that already holds a gate. */
|
|
89
|
+
export function openEvalGate(dir, config) {
|
|
90
|
+
if (!validConfig(config))
|
|
91
|
+
throw new Error("an eval gate needs a whole dispatch limit");
|
|
92
|
+
mkdirSync(join(dir, SLOTS), { recursive: false, mode: 0o700 });
|
|
93
|
+
const fd = openSync(join(dir, CONFIG), constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
94
|
+
try {
|
|
95
|
+
writeSync(fd, JSON.stringify(config));
|
|
96
|
+
}
|
|
97
|
+
finally {
|
|
98
|
+
closeSync(fd);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/** The world's gate, as this process's environment names it: null outside an eval world, its terms, or why it can't
|
|
102
|
+
* be read (which refuses every dispatch). */
|
|
103
|
+
export function evalWorldGate(env = process.env) {
|
|
104
|
+
const dir = env[EVAL_GATE_ENV];
|
|
105
|
+
if (dir === undefined || dir === "")
|
|
106
|
+
return null;
|
|
107
|
+
const bytes = readNoFollow(join(dir, CONFIG), MAX_GATE_BYTES);
|
|
108
|
+
if (bytes === null)
|
|
109
|
+
return { dir, problem: "this eval world's dispatch gate can't be read, so nothing is dispatched" };
|
|
110
|
+
let parsed;
|
|
111
|
+
try {
|
|
112
|
+
parsed = JSON.parse(bytes.toString("utf8"));
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
parsed = null;
|
|
116
|
+
}
|
|
117
|
+
return validConfig(parsed) ? { dir, config: parsed } : { dir, problem: "this eval world's dispatch gate is malformed, so nothing is dispatched" };
|
|
118
|
+
}
|
|
119
|
+
/** Take one dispatch slot of this eval world, before a provider is called. Outside an eval world: `{ active: false }`. */
|
|
120
|
+
export function takeEvalDispatch(kind, env = process.env, now = new Date()) {
|
|
121
|
+
const gate = evalWorldGate(env);
|
|
122
|
+
if (gate === null)
|
|
123
|
+
return { active: false };
|
|
124
|
+
if ("problem" in gate)
|
|
125
|
+
return { active: true, ok: false, why: gate.problem };
|
|
126
|
+
const record = JSON.stringify({ kind: kind.slice(0, 80), pid: process.pid, at: now.toISOString() });
|
|
127
|
+
for (let n = 1; n <= gate.config.max; n++) {
|
|
128
|
+
let fd;
|
|
129
|
+
try {
|
|
130
|
+
fd = openSync(join(gate.dir, SLOTS, `${n}.json`), constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
131
|
+
}
|
|
132
|
+
catch (error) {
|
|
133
|
+
if (error.code === "EEXIST")
|
|
134
|
+
continue;
|
|
135
|
+
return { active: true, ok: false, why: "this eval world's dispatch gate can't be written, so nothing is dispatched" };
|
|
136
|
+
}
|
|
137
|
+
try {
|
|
138
|
+
writeSync(fd, record);
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
closeSync(fd);
|
|
142
|
+
}
|
|
143
|
+
return { active: true, ok: true, n };
|
|
144
|
+
}
|
|
145
|
+
// Remembered for the runner: an attempt that asked past its limit is a fail, whatever it built.
|
|
146
|
+
try {
|
|
147
|
+
const fd = openSync(join(gate.dir, OVER), constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW, 0o600);
|
|
148
|
+
closeSync(fd);
|
|
149
|
+
}
|
|
150
|
+
catch { /* the slots alone still hold the limit */ }
|
|
151
|
+
return { active: true, ok: false, why: `this eval case's limit of ${gate.config.max} provider dispatches is reached (eval-dispatch-limit)` };
|
|
152
|
+
}
|
|
153
|
+
/** Whether this process is in an eval world (any gate named, readable or not): the helpers that keep no record refuse
|
|
154
|
+
* there. */
|
|
155
|
+
export function inEvalWorld(env = process.env) {
|
|
156
|
+
return evalWorldGate(env) !== null;
|
|
157
|
+
}
|
|
158
|
+
/** Whether a dispatch in this world asked past its limit (and was refused). */
|
|
159
|
+
export function evalGateOverLimit(dir) {
|
|
160
|
+
try {
|
|
161
|
+
return lstatSync(join(dir, OVER)).isFile();
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
return false;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
/** The slots a world's dispatches took, read back by the runner (never following a link). Null when the folder can't
|
|
168
|
+
* be read whole: the runner then counts the world's spending as unknown. */
|
|
169
|
+
export function evalGateSlots(dir) {
|
|
170
|
+
let names;
|
|
171
|
+
try {
|
|
172
|
+
names = readdirSync(join(dir, SLOTS));
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
const slots = [];
|
|
178
|
+
for (const name of names) {
|
|
179
|
+
const match = /^([1-9][0-9]{0,2})\.json$/.exec(name);
|
|
180
|
+
if (match === null)
|
|
181
|
+
return null;
|
|
182
|
+
const bytes = readNoFollow(join(dir, SLOTS, name), MAX_GATE_BYTES);
|
|
183
|
+
if (bytes === null)
|
|
184
|
+
return null;
|
|
185
|
+
let parsed;
|
|
186
|
+
try {
|
|
187
|
+
parsed = JSON.parse(bytes.toString("utf8"));
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)
|
|
193
|
+
|| typeof parsed["kind"] !== "string" || parsed["kind"] === ""
|
|
194
|
+
|| !Number.isSafeInteger(parsed["pid"]) || Number(parsed["pid"]) <= 0 || typeof parsed["at"] !== "string")
|
|
195
|
+
return null;
|
|
196
|
+
slots.push({ n: Number(match[1]), kind: typeof parsed["kind"] === "string" ? parsed["kind"] : "unknown", pid: Number(parsed["pid"]) || 0, at: typeof parsed["at"] === "string" ? parsed["at"] : "" });
|
|
197
|
+
}
|
|
198
|
+
return slots.sort((a, b) => a.n - b.n);
|
|
199
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/** One required call: in this file, inside this top-level function, a call to this name, the name bound as `from`
|
|
2
|
+
* says (a module specifier its un-renamed named import comes from; `self`: the file's own top-level function; `local`:
|
|
3
|
+
* declared inside `inside`). `args`: the call's arguments, as text, must stay as the base has them. */
|
|
4
|
+
export type EvalHook = {
|
|
5
|
+
file: string;
|
|
6
|
+
inside: string;
|
|
7
|
+
calls: string;
|
|
8
|
+
from: string;
|
|
9
|
+
args?: true;
|
|
10
|
+
why: string;
|
|
11
|
+
};
|
|
12
|
+
/** One function whose body (its text, as the candidate has it) must stay as the base has it: a top-level function, or
|
|
13
|
+
* `Class#method`, a method of a top-level class. */
|
|
14
|
+
export type EvalPinned = {
|
|
15
|
+
file: string;
|
|
16
|
+
fn: string;
|
|
17
|
+
why: string;
|
|
18
|
+
};
|
|
19
|
+
export declare const EVAL_HOOKS: readonly EvalHook[];
|
|
20
|
+
export declare const EVAL_PINNED: readonly EvalPinned[];
|
|
21
|
+
/** The Store methods pinned by body, which no file may write over (storeRewriteProblem, store.ts sealStoreMoneyMethods). */
|
|
22
|
+
export declare const PINNED_STORE_METHODS: readonly string[];
|
|
23
|
+
/**
|
|
24
|
+
* Why a candidate drops a required hook or changes a pinned body, or null. `candidate` gives a listed file's bytes at
|
|
25
|
+
* the candidate (null: missing or unreadable, which needs a person too). With `base` (the plane's own record of where
|
|
26
|
+
* the release started, which the candidate can't rewrite), only what the base holds is required: a project that isn't
|
|
27
|
+
* Toolroll holds none, and a hook not yet landed can't be dropped.
|
|
28
|
+
*/
|
|
29
|
+
export declare function evalHookProblem(candidate: (file: string) => Buffer | null, base?: (file: string) => Buffer | null, hooks?: readonly EvalHook[], pinned?: readonly EvalPinned[]): string | null;
|
|
30
|
+
/** Every file the manifest reads, for one verified read of each tree. */
|
|
31
|
+
export declare const EVAL_HOOK_FILES: readonly string[];
|
|
32
|
+
/**
|
|
33
|
+
* Why a changed file could write over a pinned Store method, or null (PR #263 security scan). Each changed JavaScript
|
|
34
|
+
* or TypeScript file, store.ts included (its class's own method declarations exempt; their bodies are pinned), is held to
|
|
35
|
+
* storeRewrites: a construct the base's copy of that file didn't already have (counted, by the sha256 of its whole
|
|
36
|
+
* text) needs a person. A tree or file that can't be read, or a file that doesn't parse, needs one too; `paths` are the
|
|
37
|
+
* files the verified diff says exist at the candidate, so no bytes is a failed read, never a deletion.
|
|
38
|
+
*/
|
|
39
|
+
export declare function storeRewriteProblem(paths: readonly string[], candidate: ((file: string) => Buffer | null) | null, base: ((file: string) => Buffer | null) | null, pinned?: readonly string[]): string | null;
|