toolroll 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -24
- package/dist/accent-colors.d.ts +6 -3
- package/dist/accent-colors.js +9 -4
- package/dist/browser/workspace.css +1 -1
- package/dist/browser/workspace.js +12 -12
- package/dist/browser-workspace.d.ts +12 -0
- package/dist/builder.d.ts +18 -1
- package/dist/builder.js +177 -5
- package/dist/chat-delivery.js +6 -4
- package/dist/claim.d.ts +15 -1
- package/dist/claim.js +30 -2
- package/dist/control-ui.js +1 -1
- package/dist/daemon.d.ts +9 -0
- package/dist/daemon.js +21 -0
- package/dist/decision.d.ts +6 -0
- package/dist/decision.js +12 -0
- package/dist/dispatch.d.ts +3 -1
- package/dist/dispatch.js +13 -0
- package/dist/dispose.js +11 -7
- package/dist/exhaustion.d.ts +14 -0
- package/dist/exhaustion.js +35 -0
- package/dist/invoke.js +26 -0
- package/dist/lead-status.d.ts +7 -0
- package/dist/lead-status.js +15 -2
- package/dist/mate.js +5 -2
- package/dist/operate.js +75 -13
- package/dist/provider-auth.d.ts +113 -0
- package/dist/provider-auth.js +250 -0
- package/dist/provider.d.ts +2 -0
- package/dist/provider.js +8 -1
- package/dist/serve.d.ts +2 -0
- package/dist/serve.js +118 -74
- package/dist/setup-guide.d.ts +2 -2
- package/dist/setup-guide.js +2 -0
- package/dist/skills.d.ts +6 -0
- package/dist/skills.js +8 -0
- package/dist/store.d.ts +2 -2
- package/dist/store.js +13 -4
- package/dist/subscription-chat.js +3 -0
- package/dist/work-index.js +20 -4
- package/dist/worktree.d.ts +13 -0
- package/dist/worktree.js +55 -29
- package/package.json +1 -1
package/dist/exhaustion.d.ts
CHANGED
|
@@ -20,6 +20,10 @@
|
|
|
20
20
|
* class. Exit code, bare rate-limit prose, timeout, notFound, stderr,
|
|
21
21
|
* and zero tokens NEVER independently establish exhaustion.
|
|
22
22
|
* - Everything unmatched is `unknown` — the ordinary failure road.
|
|
23
|
+
* - A sign-in that no longer works (`auth-expired`) is recognized FIRST,
|
|
24
|
+
* for every provider and version: an expired or revoked login is not a
|
|
25
|
+
* property of a CLI version, and retrying it only burns attempts. It is
|
|
26
|
+
* never fallback-eligible — a lapsed login must not reach a paid key.
|
|
23
27
|
*/
|
|
24
28
|
import type { ProviderId } from "./provider.js";
|
|
25
29
|
export type TerminalClass =
|
|
@@ -29,6 +33,9 @@ export type TerminalClass =
|
|
|
29
33
|
| "credits-depleted"
|
|
30
34
|
/** Temporary throttling NOT tied to the usage cap — backoff, NEVER paid fallback. */
|
|
31
35
|
| "transient-throttle"
|
|
36
|
+
/** The provider's sign-in or API key no longer works — NO retry, NO
|
|
37
|
+
* fallback: dispatch for that provider pauses until someone signs in. */
|
|
38
|
+
| "auth-expired"
|
|
32
39
|
/** A definite non-exhaustion terminal (ordinary failure). */
|
|
33
40
|
| "not-exhausted"
|
|
34
41
|
/** No structural signal matched — the ordinary failure/strike road. */
|
|
@@ -82,4 +89,11 @@ export declare function classifyTerminal(args: {
|
|
|
82
89
|
version: string | null;
|
|
83
90
|
authMode: "subscription" | "api-key";
|
|
84
91
|
terminal: StructuralTerminal | null;
|
|
92
|
+
/** What a run that failed within seconds, having produced nothing, said
|
|
93
|
+
* on its way out (stderr and unstructured stdout) — read ONLY for the
|
|
94
|
+
* sign-in signal: a CLI that is not logged in often exits before its
|
|
95
|
+
* structured stream begins. */
|
|
96
|
+
earlyExit?: string | null;
|
|
85
97
|
}): TerminalClass;
|
|
98
|
+
/** Whether a provider's own words say its sign-in or key no longer works. */
|
|
99
|
+
export declare function isAuthFailure(text: string): boolean;
|
package/dist/exhaustion.js
CHANGED
|
@@ -20,6 +20,10 @@
|
|
|
20
20
|
* class. Exit code, bare rate-limit prose, timeout, notFound, stderr,
|
|
21
21
|
* and zero tokens NEVER independently establish exhaustion.
|
|
22
22
|
* - Everything unmatched is `unknown` — the ordinary failure road.
|
|
23
|
+
* - A sign-in that no longer works (`auth-expired`) is recognized FIRST,
|
|
24
|
+
* for every provider and version: an expired or revoked login is not a
|
|
25
|
+
* property of a CLI version, and retrying it only burns attempts. It is
|
|
26
|
+
* never fallback-eligible — a lapsed login must not reach a paid key.
|
|
23
27
|
*/
|
|
24
28
|
/** The two classes that may authorize a fallback advance. */
|
|
25
29
|
export const FALLBACK_ELIGIBLE = new Set([
|
|
@@ -111,6 +115,12 @@ export function classMatchesAuthMode(cls, authMode) {
|
|
|
111
115
|
*/
|
|
112
116
|
export function classifyTerminal(args) {
|
|
113
117
|
const { provider, version, authMode, terminal } = args;
|
|
118
|
+
// A sign-in that no longer works, before anything else and for any
|
|
119
|
+
// provider or version: no fixture is needed to know a login is gone.
|
|
120
|
+
if (terminal !== null && terminal.failed === true && isAuthFailure(`${terminal.code ?? ""}\n${terminal.text ?? ""}`))
|
|
121
|
+
return "auth-expired";
|
|
122
|
+
if (typeof args.earlyExit === "string" && isAuthFailure(args.earlyExit))
|
|
123
|
+
return "auth-expired";
|
|
114
124
|
if (terminal === null)
|
|
115
125
|
return "unknown";
|
|
116
126
|
// The classifier's contract permits failed:false, but a non-failure
|
|
@@ -139,3 +149,28 @@ export function classifyTerminal(args) {
|
|
|
139
149
|
// unknown above).
|
|
140
150
|
return "not-exhausted";
|
|
141
151
|
}
|
|
152
|
+
/**
|
|
153
|
+
* The sign-in signals, provider-neutral: Claude's expired or unrefreshable
|
|
154
|
+
* OAuth session and its "Please run /login", an API key the provider calls
|
|
155
|
+
* invalid or revoked, Codex's and Gemini's "not logged in", and an HTTP 401
|
|
156
|
+
* the harness reports as its terminal. Ordinary failures (a test that
|
|
157
|
+
* failed, a timeout, a usage limit) match none of these.
|
|
158
|
+
*/
|
|
159
|
+
const AUTH_FAILURE = [
|
|
160
|
+
// Provider-specific phrasings only: a build's own output (a login feature's
|
|
161
|
+
// tests, an HTTP 401 in a log) must never read as the agent's sign-in.
|
|
162
|
+
/\bOAuth (?:session|token)\b[^\n]{0,80}\b(?:expired|could not be refreshed|revoked|invalid)/i,
|
|
163
|
+
/\bFailed to authenticate\b[^\n]{0,40}\b(?:OAuth|token|session|credentials|API key)/i,
|
|
164
|
+
/\bPlease run \/login\b/i,
|
|
165
|
+
/\binvalid[ _-](?:x-)?api[ _-]?key\b/i,
|
|
166
|
+
/\binvalid bearer token\b/i,
|
|
167
|
+
/\bapi[ _-]?key\b[^\n]{0,40}\b(?:is )?(?:invalid|revoked|expired|not valid)\b/i,
|
|
168
|
+
/"type"\s*:\s*"authentication_error"/i,
|
|
169
|
+
/\bMissing bearer (?:or basic )?authentication\b/i,
|
|
170
|
+
/^Not logged in\b/m,
|
|
171
|
+
];
|
|
172
|
+
/** Whether a provider's own words say its sign-in or key no longer works. */
|
|
173
|
+
export function isAuthFailure(text) {
|
|
174
|
+
const bounded = text.slice(0, 8192);
|
|
175
|
+
return AUTH_FAILURE.some(re => re.test(bounded));
|
|
176
|
+
}
|
package/dist/invoke.js
CHANGED
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
import { adapterFor, auditOf } from "./provider.js";
|
|
20
20
|
import { readProviderKey, readAuthModeStrict, PROVIDER_KEY_ENV, OWN_KEY_ENV } from "./keys.js";
|
|
21
21
|
import { classifyTerminal } from "./exhaustion.js";
|
|
22
|
+
import { liftAuthPause } from "./provider-auth.js";
|
|
22
23
|
import { attestProvider } from "./attest.js";
|
|
23
24
|
import { run as runCommand, runOwnerTag, startClaudeHeldSession } from "./exec.js";
|
|
24
25
|
import { currentContainment } from "./containment.js";
|
|
@@ -32,6 +33,9 @@ import { billingOf, claudeBillingFrom, seenBilling } from "./spend.js";
|
|
|
32
33
|
/** The claude binary name — kept for the legacy quota rows and tests that
|
|
33
34
|
* describe history; new code carries a resolved AgentSpec instead. */
|
|
34
35
|
export const PROVIDER_BINARY = "claude";
|
|
36
|
+
/** "Within seconds": how soon a silent failed run must end for its stderr to
|
|
37
|
+
* be read for the sign-in signal. */
|
|
38
|
+
const AUTH_EARLY_EXIT_MS = 30_000;
|
|
35
39
|
/**
|
|
36
40
|
* A provider can run the repository's own CLI. In a self-hosting build that
|
|
37
41
|
* must never mean "open the database that launched me": a migration from the
|
|
@@ -306,6 +310,7 @@ export async function invokeAgent(store, runId, spec, invocation, options) {
|
|
|
306
310
|
? await codexLoginBilling(attested !== null ? attested.executable : adapter.binary, [...(runOptions.omitEnv ?? []), ...adapter.extraOmitEnv, ...ownKeyEnv])
|
|
307
311
|
: null;
|
|
308
312
|
let result;
|
|
313
|
+
const spawnedAt = Date.now();
|
|
309
314
|
try {
|
|
310
315
|
// Under the stop watch (v52): the run's stop row is re-read while the
|
|
311
316
|
// provider runs, and a stop kills THIS run's process group through the
|
|
@@ -438,11 +443,20 @@ export async function invokeAgent(store, runId, spec, invocation, options) {
|
|
|
438
443
|
// no version here yet (attested === null), so they classify fail-closed
|
|
439
444
|
// until their exhaustion fixture — and the version proving it needs — is
|
|
440
445
|
// captured and reviewed.
|
|
446
|
+
// A run that failed within seconds having produced nothing is read for
|
|
447
|
+
// the sign-in signal too: a CLI that is not logged in often says so on
|
|
448
|
+
// stderr before its structured stream starts.
|
|
449
|
+
const quietEarlyExit = result.code !== 0 && !result.timedOut && !result.notFound &&
|
|
450
|
+
Date.now() - spawnedAt <= AUTH_EARLY_EXIT_MS &&
|
|
451
|
+
envelope.promptConsumed !== true && (envelope.tokensOut ?? 0) === 0;
|
|
441
452
|
const terminalClass = classifyTerminal({
|
|
442
453
|
provider: spec.provider,
|
|
443
454
|
version: attested === null ? null : attested.version,
|
|
444
455
|
authMode,
|
|
445
456
|
terminal: envelope.structuralTerminal,
|
|
457
|
+
// stderr and plain stdout only: a JSON event line can carry the agent's
|
|
458
|
+
// own command output (codex aggregated_output), never its sign-in.
|
|
459
|
+
earlyExit: quietEarlyExit ? `${result.stderr.slice(-4096)}\n${plainLines(result.stdout).slice(-4096)}` : null,
|
|
446
460
|
});
|
|
447
461
|
store.stampTerminalClass(runId, authMode, terminalClass);
|
|
448
462
|
// Contradictory structural identity is a provider protocol failure, not
|
|
@@ -519,6 +533,14 @@ export async function invokeAgent(store, runId, spec, invocation, options) {
|
|
|
519
533
|
finalMessage: envelope.finalMessage,
|
|
520
534
|
};
|
|
521
535
|
}
|
|
536
|
+
// A run that worked proves its provider's sign-in works: a sign-in pause
|
|
537
|
+
// on it lifts, and its waiting tasks may start again.
|
|
538
|
+
if (result.code === 0 && envelope.structuralTerminal === null && envelope.promptConsumed !== false) {
|
|
539
|
+
try {
|
|
540
|
+
liftAuthPause(store, spec.provider, "run", `run #${runId}`, clock(), new Date(spawnedAt));
|
|
541
|
+
}
|
|
542
|
+
catch { /* the pause stays until the next proof */ }
|
|
543
|
+
}
|
|
522
544
|
return {
|
|
523
545
|
kind: "ran",
|
|
524
546
|
outcome: {
|
|
@@ -778,3 +800,7 @@ async function codexLoginBilling(binary, omitEnv) {
|
|
|
778
800
|
return null;
|
|
779
801
|
}
|
|
780
802
|
}
|
|
803
|
+
/** The lines of a harness's stdout that are not JSON events. */
|
|
804
|
+
function plainLines(stdout) {
|
|
805
|
+
return stdout.split("\n").filter(line => !line.trimStart().startsWith("{")).join("\n");
|
|
806
|
+
}
|
package/dist/lead-status.d.ts
CHANGED
|
@@ -62,6 +62,13 @@ export type InstallationStatus = {
|
|
|
62
62
|
plan: string | null;
|
|
63
63
|
observedAt: string;
|
|
64
64
|
})[];
|
|
65
|
+
/** Providers whose sign-in stopped working: their dispatch is paused. */
|
|
66
|
+
signIn: {
|
|
67
|
+
provider: string;
|
|
68
|
+
reason: string;
|
|
69
|
+
command: string;
|
|
70
|
+
since: string;
|
|
71
|
+
}[];
|
|
65
72
|
};
|
|
66
73
|
/** One indexed observation for `task wait`. `watchedRun` is the attempt the
|
|
67
74
|
* caller first saw. Each poll follows the task's newest attempt from there, so
|
package/dist/lead-status.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* this module never opens a repository, retained evidence, or a live log. */
|
|
3
3
|
import { COMPLETION_ACTION } from "./result-completion.js";
|
|
4
4
|
import { windowLabel } from "./provider-limits.js";
|
|
5
|
+
import { openAuthPauses, signInCommand, signInReason } from "./provider-auth.js";
|
|
5
6
|
import { BUILT_IN, checkSuitesOf } from "./store.js";
|
|
6
7
|
const maybeNumber = (value) => value === null || value === undefined || !Number.isInteger(Number(value)) ? null : Number(value);
|
|
7
8
|
const CHECK_COLUMNS = "rc.status AS check_status, rc.exit_code AS check_exit_code, rc.suites_json AS check_suites_json";
|
|
@@ -192,8 +193,16 @@ export function installationStatus(store, now) {
|
|
|
192
193
|
WHERE task.state = 'queued'`).all(now.toISOString(), BUILT_IN);
|
|
193
194
|
const reasons = new Map();
|
|
194
195
|
const queuedTasks = [];
|
|
196
|
+
const pauses = openAuthPauses(store);
|
|
195
197
|
for (const row of queuedRows) {
|
|
196
|
-
|
|
198
|
+
let reason = queueReason(row);
|
|
199
|
+
if (reason === "ready for a worker" && pauses.length > 0) {
|
|
200
|
+
const scope = store.getScope(String(row["id"]));
|
|
201
|
+
const provider = scope?.approvedProfile?.provider ?? scope?.profile?.provider;
|
|
202
|
+
const paused = pauses.find(one => one.provider === provider);
|
|
203
|
+
if (paused !== undefined)
|
|
204
|
+
reason = signInReason(paused);
|
|
205
|
+
}
|
|
197
206
|
reasons.set(reason, (reasons.get(reason) ?? 0) + 1);
|
|
198
207
|
if (queuedTasks.length < 8)
|
|
199
208
|
queuedTasks.push({ task: String(row["id"]), reason });
|
|
@@ -243,10 +252,14 @@ export function installationStatus(store, now) {
|
|
|
243
252
|
reached: Number(row["reached"]) === 1,
|
|
244
253
|
observedAt: String(row["observed_at"]),
|
|
245
254
|
})),
|
|
255
|
+
signIn: pauses.map(one => ({ provider: one.provider, reason: signInReason(one), command: signInCommand(one), since: one.openedAt })),
|
|
246
256
|
};
|
|
247
257
|
}
|
|
248
258
|
export function renderInstallationStatus(status) {
|
|
249
259
|
const lines = [];
|
|
260
|
+
// A paused provider comes first: it is the one thing a person must do.
|
|
261
|
+
for (const one of status.signIn)
|
|
262
|
+
lines.push(`${one.reason} — run \`${one.command}\`. Its tasks wait until then.`);
|
|
250
263
|
lines.push(status.running.count === 0 ? "Running: none" : `Running: ${status.running.count} — ${status.running.tasks.map(one => `${one.task} (#${one.run}, ${one.phase})`).join(", ")}${status.running.count > status.running.tasks.length ? ", …" : ""}`);
|
|
251
264
|
lines.push(status.queued.count === 0 ? "Queued: none" : `Queued: ${status.queued.count} — ${status.queued.tasks.map(one => `${one.task} (${one.reason})`).join(", ")}${status.queued.count > status.queued.tasks.length ? ", …" : ""}`);
|
|
252
265
|
lines.push(status.waitingForReview.count === 0 ? "Ready for review: none" : `Ready for review: ${status.waitingForReview.count} — ${status.waitingForReview.results.map(one => `${one.task} (#${one.run})`).join(", ")}${status.waitingForReview.count > status.waitingForReview.results.length ? ", …" : ""}`);
|
|
@@ -276,7 +289,7 @@ export function renderInstallationStatus(status) {
|
|
|
276
289
|
lines.push(`Plan windows: ${provider} — ${detail}`);
|
|
277
290
|
}
|
|
278
291
|
}
|
|
279
|
-
return lines.slice(0, 12);
|
|
292
|
+
return lines.slice(0, 12 + status.signIn.length);
|
|
280
293
|
}
|
|
281
294
|
export function renderTaskWait(snapshot, outcome = snapshot.outcome) {
|
|
282
295
|
const check = snapshot.check.status === "unknown" ? "checks unknown" : snapshot.check.status === "not-run" ? "checks not run" : `checks ${snapshot.check.status}${snapshot.check.exitCode === null ? "" : ` (exit ${snapshot.check.exitCode})`}`;
|
package/dist/mate.js
CHANGED
|
@@ -410,10 +410,13 @@ export async function runMateTurn(input) {
|
|
|
410
410
|
const outcome = executeMateTool({ store, who, now: clock(), draft, selectEvidence, step: steps, readDecisions, readResults, ...(input.evidenceRoot === undefined ? {} : { evidenceRoot: input.evidenceRoot }), ...(input.mediaDelivery === undefined ? {} : { mediaDelivery: input.mediaDelivery }) }, call.name, call.args, view);
|
|
411
411
|
if (READ_TOOLS.has(call.name))
|
|
412
412
|
reads++;
|
|
413
|
-
// Opt-in, local diagnostics for end-to-end runs: what the lead asked of each tool and what came back, keys
|
|
413
|
+
// Opt-in, local diagnostics for end-to-end runs: what the lead asked of each tool and what came back (its start), keys
|
|
414
|
+
// blanked. Each line stays whole JSON so a run can assert on what a tool returned rather than on the model's words.
|
|
414
415
|
if (envValue(process.env, "MATE_TRACE") === "1") {
|
|
415
416
|
const blanked = (_, value) => typeof value === "string" ? redactSecretAssignments(redactSecretLines(value, scanForSecrets(value))) : value;
|
|
416
|
-
|
|
417
|
+
const args = JSON.stringify(call.args, blanked).slice(0, 1500);
|
|
418
|
+
const result = JSON.stringify(outcome.ok ? outcome.body : null, blanked)?.slice(0, 2000) ?? null;
|
|
419
|
+
process.stderr.write(`mate-trace ${JSON.stringify({ turn: turnId, step: steps, tool: call.name, args: args.length < 1500 ? call.args : args, ok: outcome.ok, ...(outcome.ok ? { result } : { message: outcome.message }) }, blanked)}\n`);
|
|
417
420
|
}
|
|
418
421
|
history.push({ role: "tool", callId: call.id, name: call.name, result: capped(outcome.ok ? outcome.body : { ok: false, message: outcome.message }) });
|
|
419
422
|
}
|
package/dist/operate.js
CHANGED
|
@@ -10,6 +10,8 @@ import { billingOf, budgetHoldWords, budgetLabel, budgetStates, monthNamed, mont
|
|
|
10
10
|
import { spendCsv } from "./spend-ui.js";
|
|
11
11
|
import { buildExport, exportSummary, exportZip, writeExportFolder } from "./export.js";
|
|
12
12
|
import { startBudgetAlerts } from "./budget-alerts.js";
|
|
13
|
+
import { liftAuthPause, openAuthPauses, signInGate, signInWords, startSignInProbes } from "./provider-auth.js";
|
|
14
|
+
import { createConnectionChecker } from "./provider-connection.js";
|
|
13
15
|
import { backupFiles, backupFolderOf, backupOwner, backupNow, restoreDatabase, startBackups } from "./backup.js";
|
|
14
16
|
import { pushLimitSink } from "./provider-limits.js";
|
|
15
17
|
import { limitsView } from "./limits-ui.js";
|
|
@@ -121,7 +123,7 @@ import { verdictWords as proofVerdictWords, matrixWords, semanticCoverage, cover
|
|
|
121
123
|
import { probeRepo, isVerified } from "./probe.js";
|
|
122
124
|
import { diagnoseTaskDispatch, diagnosisIsDispatchable, scopeApprovedForDispatch, } from "./dispatch.js";
|
|
123
125
|
import { createDecisionServer } from "./serve.js";
|
|
124
|
-
import { daemonLaunchCommand, daemonStatus,
|
|
126
|
+
import { daemonLaunchCommand, daemonStatus, installReplacingLegacy, installedLegacyDaemon, planDaemon, uninstallDaemon, awaitFreshHeartbeat, } from "./daemon.js";
|
|
125
127
|
import { bridgePass, clearBotToken, createTransport, followBridge, hashPairingCode, loadBotToken, mintPairingCode, redactToken, saveBotToken, PAIRING_TTL_MS, TOKEN_ENV, telegramHookSecret, telegramPushUrl, } from "./telegram.js";
|
|
126
128
|
import { scanRepo } from "./capscan.js";
|
|
127
129
|
import { computeGaps, describeCapability } from "./gaps.js";
|
|
@@ -405,6 +407,8 @@ Agents — which provider and model each phase runs on
|
|
|
405
407
|
toolroll providers --report --runner <name> --token <t>
|
|
406
408
|
record this machine's readiness per
|
|
407
409
|
provider under its runner name
|
|
410
|
+
toolroll providers resume <provider> after signing in again, start its
|
|
411
|
+
paused tasks now
|
|
408
412
|
toolroll config set chat --provider claude-subscription|codex-subscription|anthropic-api|openrouter-api
|
|
409
413
|
[--model <m>] [--weekly-usd <n>] [--daily-turns <n>] --as <you> --token <t>
|
|
410
414
|
membership providers reuse a logged-in local harness with no dollar
|
|
@@ -913,6 +917,8 @@ async function dispatch(command, positional, flags, context) {
|
|
|
913
917
|
case "intake":
|
|
914
918
|
return intakeCommand(positional, flags, context);
|
|
915
919
|
case "providers":
|
|
920
|
+
if (positional[0] === "resume")
|
|
921
|
+
return resumeProviderCommand(positional.slice(1), flags, context);
|
|
916
922
|
return providersCommand(flags, context);
|
|
917
923
|
case "template":
|
|
918
924
|
return templateCommand(positional, flags, context);
|
|
@@ -1007,6 +1013,7 @@ async function readyCommand(flags, context) {
|
|
|
1007
1013
|
const ready = store.listReady(now);
|
|
1008
1014
|
const described = ready.map(ref => describeRef(store, ref, now));
|
|
1009
1015
|
const dispatchableCount = described.filter(one => diagnosisIsDispatchable(one.dispatch)).length;
|
|
1016
|
+
const pauses = openAuthPauses(store);
|
|
1010
1017
|
if (json) {
|
|
1011
1018
|
write(envelopeJson({
|
|
1012
1019
|
ok: ready.length > 0,
|
|
@@ -1014,10 +1021,14 @@ async function readyCommand(flags, context) {
|
|
|
1014
1021
|
...(ready.length > 0 ? {} : { reason: "empty", message: "nothing is task-locally ready" }),
|
|
1015
1022
|
count: ready.length,
|
|
1016
1023
|
dispatchableCount,
|
|
1024
|
+
signIn: pauses.map(one => ({ provider: one.provider, message: signInWords(one), since: one.openedAt })),
|
|
1017
1025
|
tasks: described,
|
|
1018
1026
|
}));
|
|
1019
1027
|
return ready.length > 0 ? EXIT.ok : EXIT.refused;
|
|
1020
1028
|
}
|
|
1029
|
+
// A paused provider first: nothing on it runs until someone signs in.
|
|
1030
|
+
for (const one of pauses)
|
|
1031
|
+
write(signInWords(one));
|
|
1021
1032
|
if (ready.length === 0) {
|
|
1022
1033
|
write("Nothing is ready to dispatch.");
|
|
1023
1034
|
return EXIT.refused;
|
|
@@ -1026,7 +1037,7 @@ async function readyCommand(flags, context) {
|
|
|
1026
1037
|
for (const ref of ready) {
|
|
1027
1038
|
const task = store.getTask(ref.externalId);
|
|
1028
1039
|
const diagnosis = diagnoseTaskDispatch(store, ref.externalId, now);
|
|
1029
|
-
write(` ${ref.externalId} ${task === null ? "" : task.title}${ref.assignedRunner === null ? "" : ` (reserved for ${ref.assignedRunner})`}${diagnosis === null ? "" : ` — ${diagnosis.summary.toLowerCase()}`}`);
|
|
1040
|
+
write(` ${ref.externalId} ${task === null ? "" : task.title}${ref.assignedRunner === null ? "" : ` (reserved for ${ref.assignedRunner})`}${diagnosis === null ? "" : ` — ${diagnosis.code === "signed-out" ? diagnosis.summary : diagnosis.summary.toLowerCase()}`}`);
|
|
1030
1041
|
}
|
|
1031
1042
|
return EXIT.ok;
|
|
1032
1043
|
}
|
|
@@ -1826,6 +1837,7 @@ async function buildCommand(positional, flags, context) {
|
|
|
1826
1837
|
const broke = result.reason === "agent" ||
|
|
1827
1838
|
result.reason === "agent-reported" ||
|
|
1828
1839
|
result.reason === "no-op" ||
|
|
1840
|
+
result.reason === "no-handoff" ||
|
|
1829
1841
|
result.reason === "moved-head" ||
|
|
1830
1842
|
result.reason === "timeout" ||
|
|
1831
1843
|
result.reason === "git" ||
|
|
@@ -2114,6 +2126,13 @@ async function tickCommand(flags, context) {
|
|
|
2114
2126
|
resumed.push({ id: taskId, outcome: "skipped", reason: "budget", detail: budgetWords(laneBudget) });
|
|
2115
2127
|
continue;
|
|
2116
2128
|
}
|
|
2129
|
+
// The sign-in pause holds a resumed lane too: it waits, parked, instead of failing.
|
|
2130
|
+
const laneSignedOut = signInGate(store, [racer.provider], clock());
|
|
2131
|
+
if (laneSignedOut !== null) {
|
|
2132
|
+
backToParked();
|
|
2133
|
+
resumed.push({ id: taskId, outcome: "skipped", reason: "signed-out", detail: signInWords(laneSignedOut) });
|
|
2134
|
+
continue;
|
|
2135
|
+
}
|
|
2117
2136
|
// Legacy comparison lanes carried an absolute clock. New profiles use
|
|
2118
2137
|
// a progress watchdog and may remain alive indefinitely while useful
|
|
2119
2138
|
// work is observable; only legacy approvals retain their cumulative
|
|
@@ -2575,6 +2594,18 @@ async function tickCommand(flags, context) {
|
|
|
2575
2594
|
dispatched.push({ id, outcome: "skipped", reason: "auth-mode", detail: brokenMode.problem });
|
|
2576
2595
|
continue;
|
|
2577
2596
|
}
|
|
2597
|
+
// THE SIGN-IN PAUSE: a provider whose sign-in stopped working takes no
|
|
2598
|
+
// new work — no claim, no run, never a substitute (a lapsed login must not
|
|
2599
|
+
// reach a paid fallback) — until a run or check on it works again or a
|
|
2600
|
+
// person resumes it. Other providers keep working.
|
|
2601
|
+
// One task every AUTH_TRIAL_MS goes ahead as the trial: a sign-in check
|
|
2602
|
+
// can say "logged in" for a session that cannot refresh, so a real run
|
|
2603
|
+
// decides — it lifts the pause, or fails into the same incident.
|
|
2604
|
+
const signedOut = signInGate(store, modeProviders, clock(), ref.id);
|
|
2605
|
+
if (signedOut !== null) {
|
|
2606
|
+
dispatched.push({ id, outcome: "skipped", reason: "signed-out", detail: signInWords(signedOut) });
|
|
2607
|
+
continue;
|
|
2608
|
+
}
|
|
2578
2609
|
// The leg is the authority: what resolved must BE the leg, exactly —
|
|
2579
2610
|
// the sealed build leg, or the parked fallback entry's own pair.
|
|
2580
2611
|
const governingLeg = wantsPlan ? planLeg : parkedEntry !== null ? { provider: parkedEntry.provider, model: parkedEntry.model, chosen: "fallback" } : sealedBuildLeg;
|
|
@@ -3678,7 +3709,9 @@ async function tickCommand(flags, context) {
|
|
|
3678
3709
|
id,
|
|
3679
3710
|
outcome: "failed",
|
|
3680
3711
|
reason: disposition.sealed
|
|
3681
|
-
?
|
|
3712
|
+
? disposition.disposition === "auth-expired"
|
|
3713
|
+
? "auth-expired — requeued, sign-in needed"
|
|
3714
|
+
: `${disposition.failureClass}${disposition.disposition === "backoff" ? ` — retry ${disposition.strikes}/3` : disposition.disposition === "stalled" ? " — stalled" : ""}`
|
|
3682
3715
|
: "fenced",
|
|
3683
3716
|
...(result.ok ? {} : { detail: result.message.slice(0, 200) }),
|
|
3684
3717
|
worktree: leased.worktree.path,
|
|
@@ -3745,6 +3778,12 @@ async function tickCommand(flags, context) {
|
|
|
3745
3778
|
dispatched.push({ id: pending.taskId, outcome: "skipped", reason: "budget-unenforceable" });
|
|
3746
3779
|
continue;
|
|
3747
3780
|
}
|
|
3781
|
+
// The sign-in pause holds a fallback entry too: the admission stays pending until its provider works.
|
|
3782
|
+
const entrySignedOut = signInGate(store, [peek.profile.provider], clock(), pending.taskRef);
|
|
3783
|
+
if (entrySignedOut !== null) {
|
|
3784
|
+
dispatched.push({ id: pending.taskId, outcome: "skipped", reason: "signed-out", detail: signInWords(entrySignedOut) });
|
|
3785
|
+
continue;
|
|
3786
|
+
}
|
|
3748
3787
|
// The FALLBACK claim (finding 4): every acquireIfReady gate — task
|
|
3749
3788
|
// state, non-backoff holds, blockers, approved scope + mode belt,
|
|
3750
3789
|
// capability, capacity, and quota keyed by the PINNED credential —
|
|
@@ -3934,6 +3973,12 @@ async function tickCommand(flags, context) {
|
|
|
3934
3973
|
dispatched.push({ id: taskId, outcome: "skipped", reason: "budget", detail: budgetWords(continuationBudget) });
|
|
3935
3974
|
continue;
|
|
3936
3975
|
}
|
|
3976
|
+
// The sign-in pause holds a continuation too: its authorization stays open and it starts once the sign-in works.
|
|
3977
|
+
const continuationSignedOut = signInGate(store, [pinned.provider], clock(), parent.taskRef);
|
|
3978
|
+
if (continuationSignedOut !== null) {
|
|
3979
|
+
dispatched.push({ id: taskId, outcome: "skipped", reason: "signed-out", detail: signInWords(continuationSignedOut) });
|
|
3980
|
+
continue;
|
|
3981
|
+
}
|
|
3937
3982
|
const claimed = acquireContinuation(store, continuation, runner, { now: clock(), token, ttlMs: leaseTtlMs });
|
|
3938
3983
|
if (!claimed.ok) {
|
|
3939
3984
|
dispatched.push({ id: taskId, outcome: "skipped", reason: claimed.reason, ...("message" in claimed ? { detail: claimed.message } : {}) });
|
|
@@ -4633,6 +4678,8 @@ async function startConsole(options) {
|
|
|
4633
4678
|
server.on("close", stopMonitoring);
|
|
4634
4679
|
// v105: budget alerts at 50/80/100 %, a pass a minute.
|
|
4635
4680
|
server.on("close", startBudgetAlerts(context.store));
|
|
4681
|
+
// A paused provider's sign-in is checked every two minutes (no model, nothing spent); the pause lifts when it works.
|
|
4682
|
+
server.on("close", startSignInProbes(context.store, createConnectionChecker()));
|
|
4636
4683
|
// Sprint 8: scheduled backups, a pass a minute; its lease also tells a restore that the console is running.
|
|
4637
4684
|
server.on("close", startBackups(context.store, context.databaseFile));
|
|
4638
4685
|
server.on("close", startCodexLimits(context.store));
|
|
@@ -5009,6 +5056,27 @@ async function webhookCommand(positional, flags, context) {
|
|
|
5009
5056
|
`Send yourself a proof: toolroll webhook test`,
|
|
5010
5057
|
]);
|
|
5011
5058
|
}
|
|
5059
|
+
/**
|
|
5060
|
+
* `toolroll providers resume <provider>` — a person says the sign-in works
|
|
5061
|
+
* again (after `claude /login`, say): the provider's sign-in pause lifts, its
|
|
5062
|
+
* tasks may start, and one short message says how many resumed. A pause that
|
|
5063
|
+
* lifted by itself answers so; nothing is paused twice.
|
|
5064
|
+
*/
|
|
5065
|
+
function resumeProviderCommand(positional, flags, context) {
|
|
5066
|
+
const { store, write, json } = context;
|
|
5067
|
+
const command = "providers resume";
|
|
5068
|
+
const provider = positional[0];
|
|
5069
|
+
if (provider === undefined || !isProviderId(provider) || positional.length > 1) {
|
|
5070
|
+
return fail(write, json, command, "usage", `Use \`toolroll providers resume <${PROVIDER_IDS.join("|")}>\`.`, EXIT.usage);
|
|
5071
|
+
}
|
|
5072
|
+
const who = text(flags, "as") ?? process.env["USER"] ?? "operator";
|
|
5073
|
+
const lifted = liftAuthPause(store, provider, "person", who, context.clock());
|
|
5074
|
+
if (lifted === null)
|
|
5075
|
+
return succeed(write, json, command, { provider, resumed: 0, paused: false }, () => [`${provider} is not paused.`]);
|
|
5076
|
+
return succeed(write, json, command, { provider, resumed: lifted.resumed, paused: false }, () => [
|
|
5077
|
+
`${provider} resumed — ${lifted.resumed} ${lifted.resumed === 1 ? "task" : "tasks"} can start again.`,
|
|
5078
|
+
]);
|
|
5079
|
+
}
|
|
5012
5080
|
/**
|
|
5013
5081
|
* `toolroll providers` — identification, never integration theater.
|
|
5014
5082
|
*
|
|
@@ -6884,18 +6952,12 @@ async function daemonCommand(positional, flags, context) {
|
|
|
6884
6952
|
write("Nothing was written. Re-run without --dry-run to install.");
|
|
6885
6953
|
return EXIT.ok;
|
|
6886
6954
|
}
|
|
6887
|
-
|
|
6888
|
-
try {
|
|
6889
|
-
await uninstallDaemon(legacy, supervise);
|
|
6890
|
-
}
|
|
6891
|
-
catch (error) {
|
|
6892
|
-
return fail(write, json, "daemon install", "supervisor", `could not stop ${legacy.label}, the same watch under its older name: ${error.message}`, EXIT.failed);
|
|
6893
|
-
}
|
|
6894
|
-
}
|
|
6895
|
-
const installed = await installDaemon(plan, token, supervise);
|
|
6955
|
+
const installed = await installReplacingLegacy(plan, token, legacy, supervise);
|
|
6896
6956
|
if (!installed.ok) {
|
|
6897
6957
|
return fail(write, json, "daemon install", "supervisor", installed.message, EXIT.failed);
|
|
6898
6958
|
}
|
|
6959
|
+
// The legacy watch beat for this runner until it stopped: only a beat after that proves the new one.
|
|
6960
|
+
const readySince = legacy === null ? heartbeatBefore : store.getRunner(runnerName)?.runner.heartbeatAt ?? null;
|
|
6899
6961
|
const started = await daemonStatus(plan, supervise);
|
|
6900
6962
|
if (started.state !== "running") {
|
|
6901
6963
|
return fail(write, json, "daemon install", "not-running", `the service was installed but did not stay running (${started.detail}) — read ${plan.logPath}; no work will be claimed until this is fixed`, EXIT.failed);
|
|
@@ -6906,7 +6968,7 @@ async function daemonCommand(positional, flags, context) {
|
|
|
6906
6968
|
// readiness receipt that proves this service reached the work loop —
|
|
6907
6969
|
// a FRESH one when the service was (re)started; the one already
|
|
6908
6970
|
// standing when a healthy running service was left alone.
|
|
6909
|
-
const fresh = await awaitFreshHeartbeat(store, runnerName, installed.action === "running" ? null :
|
|
6971
|
+
const fresh = await awaitFreshHeartbeat(store, runnerName, installed.action === "running" && legacy === null ? null : readySince);
|
|
6910
6972
|
const liveRunner = fresh.ok ? store.getRunner(runnerName)?.runner ?? null : null;
|
|
6911
6973
|
if (liveRunner === null) {
|
|
6912
6974
|
const macHint = process.platform === "darwin" && repo.startsWith(join(homedir(), "Documents"))
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sign-in pauses: when a provider's sign-in (or API key) stops working, its
|
|
3
|
+
* runs fail in seconds and every retry fails the same way. So an
|
|
4
|
+
* `auth-expired` run takes no strike and no retry: its task goes back to the
|
|
5
|
+
* queue, dispatch for THAT provider pauses (the others keep working), and a
|
|
6
|
+
* person is told once per incident, on every connected messaging channel, what
|
|
7
|
+
* to run. The pause lifts when a run or a sign-in probe on that provider
|
|
8
|
+
* succeeds, or when a person resumes it; one short message says so.
|
|
9
|
+
*
|
|
10
|
+
* One row per incident. At most one open incident per provider (a partial
|
|
11
|
+
* unique index), so however many runs hit it, one notification goes out:
|
|
12
|
+
* its key is `signin:auth-expired:<provider>:<ordinal>:<person>`, the ordinal
|
|
13
|
+
* being the incident's own number for that provider.
|
|
14
|
+
*/
|
|
15
|
+
import type { Store } from "./store.js";
|
|
16
|
+
import { type ProviderId } from "./provider.js";
|
|
17
|
+
/** Every sign-in message's dedupe key starts here: `signin:<kind>:<provider>:<ordinal>:<person>`.
|
|
18
|
+
* Not a lifecycle (`life:`) key: those are progress chatter, and an undelivered
|
|
19
|
+
* one stays out of the attention list — this one needs a person. */
|
|
20
|
+
export declare const SIGN_IN_KEY_PREFIX = "signin:";
|
|
21
|
+
export declare const PROVIDER_AUTH_SCHEMA = "\nCREATE TABLE IF NOT EXISTS provider_auth_pause (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n provider TEXT NOT NULL,\n ordinal INTEGER NOT NULL,\n auth_mode TEXT NOT NULL DEFAULT 'subscription' CHECK (auth_mode IN ('subscription', 'api-key')),\n opened_at TEXT NOT NULL,\n first_run INTEGER,\n task_ref INTEGER,\n runs INTEGER NOT NULL DEFAULT 1,\n -- 1 once a sign-in probe during this pause said the provider is signed out:\n -- only a probe that then sees it signed in again may lift the pause.\n probe_saw_out INTEGER NOT NULL DEFAULT 0,\n lifted_at TEXT,\n lifted_by TEXT,\n lifted_how TEXT CHECK (lifted_how IS NULL OR lifted_how IN ('run', 'probe', 'person')),\n resumed INTEGER,\n -- The last task let through as a trial while paused: a sign-in check can\n -- say \"logged in\" for a session that no longer works, so a real run decides.\n last_trial_at TEXT,\n UNIQUE (provider, ordinal)\n);\nCREATE UNIQUE INDEX IF NOT EXISTS provider_auth_pause_open ON provider_auth_pause (provider) WHERE lifted_at IS NULL;\n";
|
|
22
|
+
export type AuthPause = {
|
|
23
|
+
id: number;
|
|
24
|
+
provider: ProviderId;
|
|
25
|
+
ordinal: number;
|
|
26
|
+
authMode: "subscription" | "api-key";
|
|
27
|
+
openedAt: string;
|
|
28
|
+
firstRun: number | null;
|
|
29
|
+
taskRef: number | null;
|
|
30
|
+
runs: number;
|
|
31
|
+
};
|
|
32
|
+
export declare function providerName(provider: string): string;
|
|
33
|
+
/** The plain reason: "Claude needs you to sign in again". */
|
|
34
|
+
export declare function signInReason(pause: Pick<AuthPause, "provider" | "authMode">): string;
|
|
35
|
+
/** What to run, as a command. */
|
|
36
|
+
export declare function signInCommand(pause: Pick<AuthPause, "provider" | "authMode">): string;
|
|
37
|
+
/** One line for status, ready and the console. */
|
|
38
|
+
export declare function signInWords(pause: AuthPause): string;
|
|
39
|
+
/** The open pause for a provider, or null. */
|
|
40
|
+
export declare function authPauseOf(store: Store, provider: string): AuthPause | null;
|
|
41
|
+
/** The open pause the dispatch gate last left this task waiting on, or null. */
|
|
42
|
+
export declare function authWaitOf(store: Store, taskRef: number): AuthPause | null;
|
|
43
|
+
/** Every open pause, oldest first. */
|
|
44
|
+
export declare function openAuthPauses(store: Store): AuthPause[];
|
|
45
|
+
/**
|
|
46
|
+
* Record an auth-expired run against its provider's pause, opening the
|
|
47
|
+
* incident (and its one notification) when none is open. Called inside the
|
|
48
|
+
* run's own failure transaction. Returns whether this run opened it.
|
|
49
|
+
*/
|
|
50
|
+
export declare function pauseForAuth(store: Store, args: {
|
|
51
|
+
provider: string;
|
|
52
|
+
authMode: "subscription" | "api-key" | null;
|
|
53
|
+
runId: number;
|
|
54
|
+
taskRef: number;
|
|
55
|
+
now: Date;
|
|
56
|
+
}): {
|
|
57
|
+
opened: boolean;
|
|
58
|
+
pause: AuthPause;
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* Lift a provider's open pause — a run or probe on it succeeded, or a person
|
|
62
|
+
* resumed it — and send the one short "signed in again" message. Returns the
|
|
63
|
+
* number of tasks its incident sent back to the queue, or null when nothing
|
|
64
|
+
* was paused.
|
|
65
|
+
*/
|
|
66
|
+
export declare function liftAuthPause(store: Store, provider: string, how: "run" | "probe" | "person", by: string, now: Date,
|
|
67
|
+
/** For a run: when it started. A run that began before the pause opened proves nothing about the sign-in now. */
|
|
68
|
+
runStartedAt?: Date): {
|
|
69
|
+
resumed: number;
|
|
70
|
+
pause: AuthPause;
|
|
71
|
+
} | null;
|
|
72
|
+
/**
|
|
73
|
+
* What a non-spending sign-in probe said. A probe lifts the pause only after
|
|
74
|
+
* one during the same pause saw the provider signed out: a CLI that reports
|
|
75
|
+
* "logged in" while its session cannot refresh must not lift and re-open the
|
|
76
|
+
* pause over and over.
|
|
77
|
+
*/
|
|
78
|
+
export declare function noteSignInProbe(store: Store, provider: string, state: string, now: Date): {
|
|
79
|
+
resumed: number;
|
|
80
|
+
} | null;
|
|
81
|
+
/** How often a paused provider lets one task through as a trial. */
|
|
82
|
+
export declare const AUTH_TRIAL_MS: number;
|
|
83
|
+
/**
|
|
84
|
+
* Whether this task may go ahead as the paused provider's trial: at most one
|
|
85
|
+
* every AUTH_TRIAL_MS. A trial that works lifts the pause (its run proves the
|
|
86
|
+
* sign-in); one that fails the same way only counts against the open
|
|
87
|
+
* incident, so nobody is told twice.
|
|
88
|
+
*/
|
|
89
|
+
export declare function claimAuthTrial(store: Store, pause: AuthPause, now: Date): boolean;
|
|
90
|
+
/**
|
|
91
|
+
* The sign-in gate every road that starts an agent asks before it claims
|
|
92
|
+
* anything: the tick's queue, and the roads beside it (fallback entries,
|
|
93
|
+
* attended continuations, contest resumes). The open pause the work waits
|
|
94
|
+
* on, or null when it may go ahead — nothing paused, or this is the pause's
|
|
95
|
+
* one trial. A waiting task is noted on its task_ref, so the work index says
|
|
96
|
+
* exactly what the gate decided; a task let through is un-noted.
|
|
97
|
+
*/
|
|
98
|
+
export declare function signInGate(store: Store, providers: readonly string[], now: Date, taskRef?: number): AuthPause | null;
|
|
99
|
+
/** The loop beside the console: every two minutes, probe each paused
|
|
100
|
+
* provider's sign-in (no model, nothing spent) and lift the pause when it
|
|
101
|
+
* works again. Nothing is probed while nothing is paused. */
|
|
102
|
+
export declare function startSignInProbes(store: Store, check: (provider: ProviderId, fresh?: boolean) => Promise<{
|
|
103
|
+
state: string;
|
|
104
|
+
}>, everyMs?: number): () => void;
|
|
105
|
+
/** The console's view of every open pause: title, command, one action. */
|
|
106
|
+
export declare function signInNotices(store: Store): {
|
|
107
|
+
provider: string;
|
|
108
|
+
title: string;
|
|
109
|
+
command: string;
|
|
110
|
+
detail: string;
|
|
111
|
+
resumeLabel: string;
|
|
112
|
+
resumeHref: string;
|
|
113
|
+
}[];
|