toolroll 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,6 +20,10 @@
20
20
  * class. Exit code, bare rate-limit prose, timeout, notFound, stderr,
21
21
  * and zero tokens NEVER independently establish exhaustion.
22
22
  * - Everything unmatched is `unknown` — the ordinary failure road.
23
+ * - A sign-in that no longer works (`auth-expired`) is recognized FIRST,
24
+ * for every provider and version: an expired or revoked login is not a
25
+ * property of a CLI version, and retrying it only burns attempts. It is
26
+ * never fallback-eligible — a lapsed login must not reach a paid key.
23
27
  */
24
28
  import type { ProviderId } from "./provider.js";
25
29
  export type TerminalClass =
@@ -29,6 +33,9 @@ export type TerminalClass =
29
33
  | "credits-depleted"
30
34
  /** Temporary throttling NOT tied to the usage cap — backoff, NEVER paid fallback. */
31
35
  | "transient-throttle"
36
+ /** The provider's sign-in or API key no longer works — NO retry, NO
37
+ * fallback: dispatch for that provider pauses until someone signs in. */
38
+ | "auth-expired"
32
39
  /** A definite non-exhaustion terminal (ordinary failure). */
33
40
  | "not-exhausted"
34
41
  /** No structural signal matched — the ordinary failure/strike road. */
@@ -82,4 +89,11 @@ export declare function classifyTerminal(args: {
82
89
  version: string | null;
83
90
  authMode: "subscription" | "api-key";
84
91
  terminal: StructuralTerminal | null;
92
+ /** What a run that failed within seconds, having produced nothing, said
93
+ * on its way out (stderr and unstructured stdout) — read ONLY for the
94
+ * sign-in signal: a CLI that is not logged in often exits before its
95
+ * structured stream begins. */
96
+ earlyExit?: string | null;
85
97
  }): TerminalClass;
98
+ /** Whether a provider's own words say its sign-in or key no longer works. */
99
+ export declare function isAuthFailure(text: string): boolean;
@@ -20,6 +20,10 @@
20
20
  * class. Exit code, bare rate-limit prose, timeout, notFound, stderr,
21
21
  * and zero tokens NEVER independently establish exhaustion.
22
22
  * - Everything unmatched is `unknown` — the ordinary failure road.
23
+ * - A sign-in that no longer works (`auth-expired`) is recognized FIRST,
24
+ * for every provider and version: an expired or revoked login is not a
25
+ * property of a CLI version, and retrying it only burns attempts. It is
26
+ * never fallback-eligible — a lapsed login must not reach a paid key.
23
27
  */
24
28
  /** The two classes that may authorize a fallback advance. */
25
29
  export const FALLBACK_ELIGIBLE = new Set([
@@ -111,6 +115,12 @@ export function classMatchesAuthMode(cls, authMode) {
111
115
  */
112
116
  export function classifyTerminal(args) {
113
117
  const { provider, version, authMode, terminal } = args;
118
+ // A sign-in that no longer works, before anything else and for any
119
+ // provider or version: no fixture is needed to know a login is gone.
120
+ if (terminal !== null && terminal.failed === true && isAuthFailure(`${terminal.code ?? ""}\n${terminal.text ?? ""}`))
121
+ return "auth-expired";
122
+ if (typeof args.earlyExit === "string" && isAuthFailure(args.earlyExit))
123
+ return "auth-expired";
114
124
  if (terminal === null)
115
125
  return "unknown";
116
126
  // The classifier's contract permits failed:false, but a non-failure
@@ -139,3 +149,28 @@ export function classifyTerminal(args) {
139
149
  // unknown above).
140
150
  return "not-exhausted";
141
151
  }
152
+ /**
153
+ * The sign-in signals, provider-neutral: Claude's expired or unrefreshable
154
+ * OAuth session and its "Please run /login", an API key the provider calls
155
+ * invalid or revoked, Codex's and Gemini's "not logged in", and an HTTP 401
156
+ * the harness reports as its terminal. Ordinary failures (a test that
157
+ * failed, a timeout, a usage limit) match none of these.
158
+ */
159
+ const AUTH_FAILURE = [
160
+ // Provider-specific phrasings only: a build's own output (a login feature's
161
+ // tests, an HTTP 401 in a log) must never read as the agent's sign-in.
162
+ /\bOAuth (?:session|token)\b[^\n]{0,80}\b(?:expired|could not be refreshed|revoked|invalid)/i,
163
+ /\bFailed to authenticate\b[^\n]{0,40}\b(?:OAuth|token|session|credentials|API key)/i,
164
+ /\bPlease run \/login\b/i,
165
+ /\binvalid[ _-](?:x-)?api[ _-]?key\b/i,
166
+ /\binvalid bearer token\b/i,
167
+ /\bapi[ _-]?key\b[^\n]{0,40}\b(?:is )?(?:invalid|revoked|expired|not valid)\b/i,
168
+ /"type"\s*:\s*"authentication_error"/i,
169
+ /\bMissing bearer (?:or basic )?authentication\b/i,
170
+ /^Not logged in\b/m,
171
+ ];
172
+ /** Whether a provider's own words say its sign-in or key no longer works. */
173
+ export function isAuthFailure(text) {
174
+ const bounded = text.slice(0, 8192);
175
+ return AUTH_FAILURE.some(re => re.test(bounded));
176
+ }
package/dist/invoke.js CHANGED
@@ -19,6 +19,7 @@
19
19
  import { adapterFor, auditOf } from "./provider.js";
20
20
  import { readProviderKey, readAuthModeStrict, PROVIDER_KEY_ENV, OWN_KEY_ENV } from "./keys.js";
21
21
  import { classifyTerminal } from "./exhaustion.js";
22
+ import { liftAuthPause } from "./provider-auth.js";
22
23
  import { attestProvider } from "./attest.js";
23
24
  import { run as runCommand, runOwnerTag, startClaudeHeldSession } from "./exec.js";
24
25
  import { currentContainment } from "./containment.js";
@@ -32,6 +33,9 @@ import { billingOf, claudeBillingFrom, seenBilling } from "./spend.js";
32
33
  /** The claude binary name — kept for the legacy quota rows and tests that
33
34
  * describe history; new code carries a resolved AgentSpec instead. */
34
35
  export const PROVIDER_BINARY = "claude";
36
+ /** "Within seconds": how soon a silent failed run must end for its stderr to
37
+ * be read for the sign-in signal. */
38
+ const AUTH_EARLY_EXIT_MS = 30_000;
35
39
  /**
36
40
  * A provider can run the repository's own CLI. In a self-hosting build that
37
41
  * must never mean "open the database that launched me": a migration from the
@@ -306,6 +310,7 @@ export async function invokeAgent(store, runId, spec, invocation, options) {
306
310
  ? await codexLoginBilling(attested !== null ? attested.executable : adapter.binary, [...(runOptions.omitEnv ?? []), ...adapter.extraOmitEnv, ...ownKeyEnv])
307
311
  : null;
308
312
  let result;
313
+ const spawnedAt = Date.now();
309
314
  try {
310
315
  // Under the stop watch (v52): the run's stop row is re-read while the
311
316
  // provider runs, and a stop kills THIS run's process group through the
@@ -438,11 +443,20 @@ export async function invokeAgent(store, runId, spec, invocation, options) {
438
443
  // no version here yet (attested === null), so they classify fail-closed
439
444
  // until their exhaustion fixture — and the version proving it needs — is
440
445
  // captured and reviewed.
446
+ // A run that failed within seconds having produced nothing is read for
447
+ // the sign-in signal too: a CLI that is not logged in often says so on
448
+ // stderr before its structured stream starts.
449
+ const quietEarlyExit = result.code !== 0 && !result.timedOut && !result.notFound &&
450
+ Date.now() - spawnedAt <= AUTH_EARLY_EXIT_MS &&
451
+ envelope.promptConsumed !== true && (envelope.tokensOut ?? 0) === 0;
441
452
  const terminalClass = classifyTerminal({
442
453
  provider: spec.provider,
443
454
  version: attested === null ? null : attested.version,
444
455
  authMode,
445
456
  terminal: envelope.structuralTerminal,
457
+ // stderr and plain stdout only: a JSON event line can carry the agent's
458
+ // own command output (codex aggregated_output), never its sign-in.
459
+ earlyExit: quietEarlyExit ? `${result.stderr.slice(-4096)}\n${plainLines(result.stdout).slice(-4096)}` : null,
446
460
  });
447
461
  store.stampTerminalClass(runId, authMode, terminalClass);
448
462
  // Contradictory structural identity is a provider protocol failure, not
@@ -519,6 +533,14 @@ export async function invokeAgent(store, runId, spec, invocation, options) {
519
533
  finalMessage: envelope.finalMessage,
520
534
  };
521
535
  }
536
+ // A run that worked proves its provider's sign-in works: a sign-in pause
537
+ // on it lifts, and its waiting tasks may start again.
538
+ if (result.code === 0 && envelope.structuralTerminal === null && envelope.promptConsumed !== false) {
539
+ try {
540
+ liftAuthPause(store, spec.provider, "run", `run #${runId}`, clock(), new Date(spawnedAt));
541
+ }
542
+ catch { /* the pause stays until the next proof */ }
543
+ }
522
544
  return {
523
545
  kind: "ran",
524
546
  outcome: {
@@ -778,3 +800,7 @@ async function codexLoginBilling(binary, omitEnv) {
778
800
  return null;
779
801
  }
780
802
  }
803
+ /** The lines of a harness's stdout that are not JSON events. */
804
+ function plainLines(stdout) {
805
+ return stdout.split("\n").filter(line => !line.trimStart().startsWith("{")).join("\n");
806
+ }
@@ -62,6 +62,13 @@ export type InstallationStatus = {
62
62
  plan: string | null;
63
63
  observedAt: string;
64
64
  })[];
65
+ /** Providers whose sign-in stopped working: their dispatch is paused. */
66
+ signIn: {
67
+ provider: string;
68
+ reason: string;
69
+ command: string;
70
+ since: string;
71
+ }[];
65
72
  };
66
73
  /** One indexed observation for `task wait`. `watchedRun` is the attempt the
67
74
  * caller first saw. Each poll follows the task's newest attempt from there, so
@@ -2,6 +2,7 @@
2
2
  * this module never opens a repository, retained evidence, or a live log. */
3
3
  import { COMPLETION_ACTION } from "./result-completion.js";
4
4
  import { windowLabel } from "./provider-limits.js";
5
+ import { openAuthPauses, signInCommand, signInReason } from "./provider-auth.js";
5
6
  import { BUILT_IN, checkSuitesOf } from "./store.js";
6
7
  const maybeNumber = (value) => value === null || value === undefined || !Number.isInteger(Number(value)) ? null : Number(value);
7
8
  const CHECK_COLUMNS = "rc.status AS check_status, rc.exit_code AS check_exit_code, rc.suites_json AS check_suites_json";
@@ -192,8 +193,16 @@ export function installationStatus(store, now) {
192
193
  WHERE task.state = 'queued'`).all(now.toISOString(), BUILT_IN);
193
194
  const reasons = new Map();
194
195
  const queuedTasks = [];
196
+ const pauses = openAuthPauses(store);
195
197
  for (const row of queuedRows) {
196
- const reason = queueReason(row);
198
+ let reason = queueReason(row);
199
+ if (reason === "ready for a worker" && pauses.length > 0) {
200
+ const scope = store.getScope(String(row["id"]));
201
+ const provider = scope?.approvedProfile?.provider ?? scope?.profile?.provider;
202
+ const paused = pauses.find(one => one.provider === provider);
203
+ if (paused !== undefined)
204
+ reason = signInReason(paused);
205
+ }
197
206
  reasons.set(reason, (reasons.get(reason) ?? 0) + 1);
198
207
  if (queuedTasks.length < 8)
199
208
  queuedTasks.push({ task: String(row["id"]), reason });
@@ -243,10 +252,14 @@ export function installationStatus(store, now) {
243
252
  reached: Number(row["reached"]) === 1,
244
253
  observedAt: String(row["observed_at"]),
245
254
  })),
255
+ signIn: pauses.map(one => ({ provider: one.provider, reason: signInReason(one), command: signInCommand(one), since: one.openedAt })),
246
256
  };
247
257
  }
248
258
  export function renderInstallationStatus(status) {
249
259
  const lines = [];
260
+ // A paused provider comes first: it is the one thing a person must do.
261
+ for (const one of status.signIn)
262
+ lines.push(`${one.reason} — run \`${one.command}\`. Its tasks wait until then.`);
250
263
  lines.push(status.running.count === 0 ? "Running: none" : `Running: ${status.running.count} — ${status.running.tasks.map(one => `${one.task} (#${one.run}, ${one.phase})`).join(", ")}${status.running.count > status.running.tasks.length ? ", …" : ""}`);
251
264
  lines.push(status.queued.count === 0 ? "Queued: none" : `Queued: ${status.queued.count} — ${status.queued.tasks.map(one => `${one.task} (${one.reason})`).join(", ")}${status.queued.count > status.queued.tasks.length ? ", …" : ""}`);
252
265
  lines.push(status.waitingForReview.count === 0 ? "Ready for review: none" : `Ready for review: ${status.waitingForReview.count} — ${status.waitingForReview.results.map(one => `${one.task} (#${one.run})`).join(", ")}${status.waitingForReview.count > status.waitingForReview.results.length ? ", …" : ""}`);
@@ -276,7 +289,7 @@ export function renderInstallationStatus(status) {
276
289
  lines.push(`Plan windows: ${provider} — ${detail}`);
277
290
  }
278
291
  }
279
- return lines.slice(0, 12);
292
+ return lines.slice(0, 12 + status.signIn.length);
280
293
  }
281
294
  export function renderTaskWait(snapshot, outcome = snapshot.outcome) {
282
295
  const check = snapshot.check.status === "unknown" ? "checks unknown" : snapshot.check.status === "not-run" ? "checks not run" : `checks ${snapshot.check.status}${snapshot.check.exitCode === null ? "" : ` (exit ${snapshot.check.exitCode})`}`;
package/dist/mate.js CHANGED
@@ -410,10 +410,13 @@ export async function runMateTurn(input) {
410
410
  const outcome = executeMateTool({ store, who, now: clock(), draft, selectEvidence, step: steps, readDecisions, readResults, ...(input.evidenceRoot === undefined ? {} : { evidenceRoot: input.evidenceRoot }), ...(input.mediaDelivery === undefined ? {} : { mediaDelivery: input.mediaDelivery }) }, call.name, call.args, view);
411
411
  if (READ_TOOLS.has(call.name))
412
412
  reads++;
413
- // Opt-in, local diagnostics for end-to-end runs: what the lead asked of each tool and what came back, keys blanked.
413
+ // Opt-in, local diagnostics for end-to-end runs: what the lead asked of each tool and what came back (its start), keys
414
+ // blanked. Each line stays whole JSON so a run can assert on what a tool returned rather than on the model's words.
414
415
  if (envValue(process.env, "MATE_TRACE") === "1") {
415
416
  const blanked = (_, value) => typeof value === "string" ? redactSecretAssignments(redactSecretLines(value, scanForSecrets(value))) : value;
416
- process.stderr.write(`mate-trace ${JSON.stringify({ turn: turnId, step: steps, tool: call.name, args: call.args, ok: outcome.ok, ...(outcome.ok ? {} : { message: outcome.message }) }, blanked).slice(0, 4000)}\n`);
417
+ const args = JSON.stringify(call.args, blanked).slice(0, 1500);
418
+ const result = JSON.stringify(outcome.ok ? outcome.body : null, blanked)?.slice(0, 2000) ?? null;
419
+ process.stderr.write(`mate-trace ${JSON.stringify({ turn: turnId, step: steps, tool: call.name, args: args.length < 1500 ? call.args : args, ok: outcome.ok, ...(outcome.ok ? { result } : { message: outcome.message }) }, blanked)}\n`);
417
420
  }
418
421
  history.push({ role: "tool", callId: call.id, name: call.name, result: capped(outcome.ok ? outcome.body : { ok: false, message: outcome.message }) });
419
422
  }
package/dist/operate.js CHANGED
@@ -10,6 +10,8 @@ import { billingOf, budgetHoldWords, budgetLabel, budgetStates, monthNamed, mont
10
10
  import { spendCsv } from "./spend-ui.js";
11
11
  import { buildExport, exportSummary, exportZip, writeExportFolder } from "./export.js";
12
12
  import { startBudgetAlerts } from "./budget-alerts.js";
13
+ import { liftAuthPause, openAuthPauses, signInGate, signInWords, startSignInProbes } from "./provider-auth.js";
14
+ import { createConnectionChecker } from "./provider-connection.js";
13
15
  import { backupFiles, backupFolderOf, backupOwner, backupNow, restoreDatabase, startBackups } from "./backup.js";
14
16
  import { pushLimitSink } from "./provider-limits.js";
15
17
  import { limitsView } from "./limits-ui.js";
@@ -121,7 +123,7 @@ import { verdictWords as proofVerdictWords, matrixWords, semanticCoverage, cover
121
123
  import { probeRepo, isVerified } from "./probe.js";
122
124
  import { diagnoseTaskDispatch, diagnosisIsDispatchable, scopeApprovedForDispatch, } from "./dispatch.js";
123
125
  import { createDecisionServer } from "./serve.js";
124
- import { daemonLaunchCommand, daemonStatus, installDaemon, installedLegacyDaemon, planDaemon, uninstallDaemon, awaitFreshHeartbeat, } from "./daemon.js";
126
+ import { daemonLaunchCommand, daemonStatus, installReplacingLegacy, installedLegacyDaemon, planDaemon, uninstallDaemon, awaitFreshHeartbeat, } from "./daemon.js";
125
127
  import { bridgePass, clearBotToken, createTransport, followBridge, hashPairingCode, loadBotToken, mintPairingCode, redactToken, saveBotToken, PAIRING_TTL_MS, TOKEN_ENV, telegramHookSecret, telegramPushUrl, } from "./telegram.js";
126
128
  import { scanRepo } from "./capscan.js";
127
129
  import { computeGaps, describeCapability } from "./gaps.js";
@@ -405,6 +407,8 @@ Agents — which provider and model each phase runs on
405
407
  toolroll providers --report --runner <name> --token <t>
406
408
  record this machine's readiness per
407
409
  provider under its runner name
410
+ toolroll providers resume <provider> after signing in again, start its
411
+ paused tasks now
408
412
  toolroll config set chat --provider claude-subscription|codex-subscription|anthropic-api|openrouter-api
409
413
  [--model <m>] [--weekly-usd <n>] [--daily-turns <n>] --as <you> --token <t>
410
414
  membership providers reuse a logged-in local harness with no dollar
@@ -913,6 +917,8 @@ async function dispatch(command, positional, flags, context) {
913
917
  case "intake":
914
918
  return intakeCommand(positional, flags, context);
915
919
  case "providers":
920
+ if (positional[0] === "resume")
921
+ return resumeProviderCommand(positional.slice(1), flags, context);
916
922
  return providersCommand(flags, context);
917
923
  case "template":
918
924
  return templateCommand(positional, flags, context);
@@ -1007,6 +1013,7 @@ async function readyCommand(flags, context) {
1007
1013
  const ready = store.listReady(now);
1008
1014
  const described = ready.map(ref => describeRef(store, ref, now));
1009
1015
  const dispatchableCount = described.filter(one => diagnosisIsDispatchable(one.dispatch)).length;
1016
+ const pauses = openAuthPauses(store);
1010
1017
  if (json) {
1011
1018
  write(envelopeJson({
1012
1019
  ok: ready.length > 0,
@@ -1014,10 +1021,14 @@ async function readyCommand(flags, context) {
1014
1021
  ...(ready.length > 0 ? {} : { reason: "empty", message: "nothing is task-locally ready" }),
1015
1022
  count: ready.length,
1016
1023
  dispatchableCount,
1024
+ signIn: pauses.map(one => ({ provider: one.provider, message: signInWords(one), since: one.openedAt })),
1017
1025
  tasks: described,
1018
1026
  }));
1019
1027
  return ready.length > 0 ? EXIT.ok : EXIT.refused;
1020
1028
  }
1029
+ // A paused provider first: nothing on it runs until someone signs in.
1030
+ for (const one of pauses)
1031
+ write(signInWords(one));
1021
1032
  if (ready.length === 0) {
1022
1033
  write("Nothing is ready to dispatch.");
1023
1034
  return EXIT.refused;
@@ -1026,7 +1037,7 @@ async function readyCommand(flags, context) {
1026
1037
  for (const ref of ready) {
1027
1038
  const task = store.getTask(ref.externalId);
1028
1039
  const diagnosis = diagnoseTaskDispatch(store, ref.externalId, now);
1029
- write(` ${ref.externalId} ${task === null ? "" : task.title}${ref.assignedRunner === null ? "" : ` (reserved for ${ref.assignedRunner})`}${diagnosis === null ? "" : ` — ${diagnosis.summary.toLowerCase()}`}`);
1040
+ write(` ${ref.externalId} ${task === null ? "" : task.title}${ref.assignedRunner === null ? "" : ` (reserved for ${ref.assignedRunner})`}${diagnosis === null ? "" : ` — ${diagnosis.code === "signed-out" ? diagnosis.summary : diagnosis.summary.toLowerCase()}`}`);
1030
1041
  }
1031
1042
  return EXIT.ok;
1032
1043
  }
@@ -1826,6 +1837,7 @@ async function buildCommand(positional, flags, context) {
1826
1837
  const broke = result.reason === "agent" ||
1827
1838
  result.reason === "agent-reported" ||
1828
1839
  result.reason === "no-op" ||
1840
+ result.reason === "no-handoff" ||
1829
1841
  result.reason === "moved-head" ||
1830
1842
  result.reason === "timeout" ||
1831
1843
  result.reason === "git" ||
@@ -2114,6 +2126,13 @@ async function tickCommand(flags, context) {
2114
2126
  resumed.push({ id: taskId, outcome: "skipped", reason: "budget", detail: budgetWords(laneBudget) });
2115
2127
  continue;
2116
2128
  }
2129
+ // The sign-in pause holds a resumed lane too: it waits, parked, instead of failing.
2130
+ const laneSignedOut = signInGate(store, [racer.provider], clock());
2131
+ if (laneSignedOut !== null) {
2132
+ backToParked();
2133
+ resumed.push({ id: taskId, outcome: "skipped", reason: "signed-out", detail: signInWords(laneSignedOut) });
2134
+ continue;
2135
+ }
2117
2136
  // Legacy comparison lanes carried an absolute clock. New profiles use
2118
2137
  // a progress watchdog and may remain alive indefinitely while useful
2119
2138
  // work is observable; only legacy approvals retain their cumulative
@@ -2575,6 +2594,18 @@ async function tickCommand(flags, context) {
2575
2594
  dispatched.push({ id, outcome: "skipped", reason: "auth-mode", detail: brokenMode.problem });
2576
2595
  continue;
2577
2596
  }
2597
+ // THE SIGN-IN PAUSE: a provider whose sign-in stopped working takes no
2598
+ // new work — no claim, no run, never a substitute (a lapsed login must not
2599
+ // reach a paid fallback) — until a run or check on it works again or a
2600
+ // person resumes it. Other providers keep working.
2601
+ // One task every AUTH_TRIAL_MS goes ahead as the trial: a sign-in check
2602
+ // can say "logged in" for a session that cannot refresh, so a real run
2603
+ // decides — it lifts the pause, or fails into the same incident.
2604
+ const signedOut = signInGate(store, modeProviders, clock(), ref.id);
2605
+ if (signedOut !== null) {
2606
+ dispatched.push({ id, outcome: "skipped", reason: "signed-out", detail: signInWords(signedOut) });
2607
+ continue;
2608
+ }
2578
2609
  // The leg is the authority: what resolved must BE the leg, exactly —
2579
2610
  // the sealed build leg, or the parked fallback entry's own pair.
2580
2611
  const governingLeg = wantsPlan ? planLeg : parkedEntry !== null ? { provider: parkedEntry.provider, model: parkedEntry.model, chosen: "fallback" } : sealedBuildLeg;
@@ -3678,7 +3709,9 @@ async function tickCommand(flags, context) {
3678
3709
  id,
3679
3710
  outcome: "failed",
3680
3711
  reason: disposition.sealed
3681
- ? `${disposition.failureClass}${disposition.disposition === "backoff" ? ` — retry ${disposition.strikes}/3` : disposition.disposition === "stalled" ? " — stalled" : ""}`
3712
+ ? disposition.disposition === "auth-expired"
3713
+ ? "auth-expired — requeued, sign-in needed"
3714
+ : `${disposition.failureClass}${disposition.disposition === "backoff" ? ` — retry ${disposition.strikes}/3` : disposition.disposition === "stalled" ? " — stalled" : ""}`
3682
3715
  : "fenced",
3683
3716
  ...(result.ok ? {} : { detail: result.message.slice(0, 200) }),
3684
3717
  worktree: leased.worktree.path,
@@ -3745,6 +3778,12 @@ async function tickCommand(flags, context) {
3745
3778
  dispatched.push({ id: pending.taskId, outcome: "skipped", reason: "budget-unenforceable" });
3746
3779
  continue;
3747
3780
  }
3781
+ // The sign-in pause holds a fallback entry too: the admission stays pending until its provider works.
3782
+ const entrySignedOut = signInGate(store, [peek.profile.provider], clock(), pending.taskRef);
3783
+ if (entrySignedOut !== null) {
3784
+ dispatched.push({ id: pending.taskId, outcome: "skipped", reason: "signed-out", detail: signInWords(entrySignedOut) });
3785
+ continue;
3786
+ }
3748
3787
  // The FALLBACK claim (finding 4): every acquireIfReady gate — task
3749
3788
  // state, non-backoff holds, blockers, approved scope + mode belt,
3750
3789
  // capability, capacity, and quota keyed by the PINNED credential —
@@ -3934,6 +3973,12 @@ async function tickCommand(flags, context) {
3934
3973
  dispatched.push({ id: taskId, outcome: "skipped", reason: "budget", detail: budgetWords(continuationBudget) });
3935
3974
  continue;
3936
3975
  }
3976
+ // The sign-in pause holds a continuation too: its authorization stays open and it starts once the sign-in works.
3977
+ const continuationSignedOut = signInGate(store, [pinned.provider], clock(), parent.taskRef);
3978
+ if (continuationSignedOut !== null) {
3979
+ dispatched.push({ id: taskId, outcome: "skipped", reason: "signed-out", detail: signInWords(continuationSignedOut) });
3980
+ continue;
3981
+ }
3937
3982
  const claimed = acquireContinuation(store, continuation, runner, { now: clock(), token, ttlMs: leaseTtlMs });
3938
3983
  if (!claimed.ok) {
3939
3984
  dispatched.push({ id: taskId, outcome: "skipped", reason: claimed.reason, ...("message" in claimed ? { detail: claimed.message } : {}) });
@@ -4633,6 +4678,8 @@ async function startConsole(options) {
4633
4678
  server.on("close", stopMonitoring);
4634
4679
  // v105: budget alerts at 50/80/100 %, a pass a minute.
4635
4680
  server.on("close", startBudgetAlerts(context.store));
4681
+ // A paused provider's sign-in is checked every two minutes (no model, nothing spent); the pause lifts when it works.
4682
+ server.on("close", startSignInProbes(context.store, createConnectionChecker()));
4636
4683
  // Sprint 8: scheduled backups, a pass a minute; its lease also tells a restore that the console is running.
4637
4684
  server.on("close", startBackups(context.store, context.databaseFile));
4638
4685
  server.on("close", startCodexLimits(context.store));
@@ -5009,6 +5056,27 @@ async function webhookCommand(positional, flags, context) {
5009
5056
  `Send yourself a proof: toolroll webhook test`,
5010
5057
  ]);
5011
5058
  }
5059
+ /**
5060
+ * `toolroll providers resume <provider>` — a person says the sign-in works
5061
+ * again (after `claude /login`, say): the provider's sign-in pause lifts, its
5062
+ * tasks may start, and one short message says how many resumed. A pause that
5063
+ * lifted by itself answers so; nothing is paused twice.
5064
+ */
5065
+ function resumeProviderCommand(positional, flags, context) {
5066
+ const { store, write, json } = context;
5067
+ const command = "providers resume";
5068
+ const provider = positional[0];
5069
+ if (provider === undefined || !isProviderId(provider) || positional.length > 1) {
5070
+ return fail(write, json, command, "usage", `Use \`toolroll providers resume <${PROVIDER_IDS.join("|")}>\`.`, EXIT.usage);
5071
+ }
5072
+ const who = text(flags, "as") ?? process.env["USER"] ?? "operator";
5073
+ const lifted = liftAuthPause(store, provider, "person", who, context.clock());
5074
+ if (lifted === null)
5075
+ return succeed(write, json, command, { provider, resumed: 0, paused: false }, () => [`${provider} is not paused.`]);
5076
+ return succeed(write, json, command, { provider, resumed: lifted.resumed, paused: false }, () => [
5077
+ `${provider} resumed — ${lifted.resumed} ${lifted.resumed === 1 ? "task" : "tasks"} can start again.`,
5078
+ ]);
5079
+ }
5012
5080
  /**
5013
5081
  * `toolroll providers` — identification, never integration theater.
5014
5082
  *
@@ -6884,18 +6952,12 @@ async function daemonCommand(positional, flags, context) {
6884
6952
  write("Nothing was written. Re-run without --dry-run to install.");
6885
6953
  return EXIT.ok;
6886
6954
  }
6887
- if (legacy !== null) {
6888
- try {
6889
- await uninstallDaemon(legacy, supervise);
6890
- }
6891
- catch (error) {
6892
- return fail(write, json, "daemon install", "supervisor", `could not stop ${legacy.label}, the same watch under its older name: ${error.message}`, EXIT.failed);
6893
- }
6894
- }
6895
- const installed = await installDaemon(plan, token, supervise);
6955
+ const installed = await installReplacingLegacy(plan, token, legacy, supervise);
6896
6956
  if (!installed.ok) {
6897
6957
  return fail(write, json, "daemon install", "supervisor", installed.message, EXIT.failed);
6898
6958
  }
6959
+ // The legacy watch beat for this runner until it stopped: only a beat after that proves the new one.
6960
+ const readySince = legacy === null ? heartbeatBefore : store.getRunner(runnerName)?.runner.heartbeatAt ?? null;
6899
6961
  const started = await daemonStatus(plan, supervise);
6900
6962
  if (started.state !== "running") {
6901
6963
  return fail(write, json, "daemon install", "not-running", `the service was installed but did not stay running (${started.detail}) — read ${plan.logPath}; no work will be claimed until this is fixed`, EXIT.failed);
@@ -6906,7 +6968,7 @@ async function daemonCommand(positional, flags, context) {
6906
6968
  // readiness receipt that proves this service reached the work loop —
6907
6969
  // a FRESH one when the service was (re)started; the one already
6908
6970
  // standing when a healthy running service was left alone.
6909
- const fresh = await awaitFreshHeartbeat(store, runnerName, installed.action === "running" ? null : heartbeatBefore);
6971
+ const fresh = await awaitFreshHeartbeat(store, runnerName, installed.action === "running" && legacy === null ? null : readySince);
6910
6972
  const liveRunner = fresh.ok ? store.getRunner(runnerName)?.runner ?? null : null;
6911
6973
  if (liveRunner === null) {
6912
6974
  const macHint = process.platform === "darwin" && repo.startsWith(join(homedir(), "Documents"))
@@ -0,0 +1,113 @@
1
+ /**
2
+ * Sign-in pauses: when a provider's sign-in (or API key) stops working, its
3
+ * runs fail in seconds and every retry fails the same way. So an
4
+ * `auth-expired` run takes no strike and no retry: its task goes back to the
5
+ * queue, dispatch for THAT provider pauses (the others keep working), and a
6
+ * person is told once per incident, on every connected messaging channel, what
7
+ * to run. The pause lifts when a run or a sign-in probe on that provider
8
+ * succeeds, or when a person resumes it; one short message says so.
9
+ *
10
+ * One row per incident. At most one open incident per provider (a partial
11
+ * unique index), so however many runs hit it, one notification goes out:
12
+ * its key is `signin:auth-expired:<provider>:<ordinal>:<person>`, the ordinal
13
+ * being the incident's own number for that provider.
14
+ */
15
+ import type { Store } from "./store.js";
16
+ import { type ProviderId } from "./provider.js";
17
+ /** Every sign-in message's dedupe key starts here: `signin:<kind>:<provider>:<ordinal>:<person>`.
18
+ * Not a lifecycle (`life:`) key: those are progress chatter, and an undelivered
19
+ * one stays out of the attention list — this one needs a person. */
20
+ export declare const SIGN_IN_KEY_PREFIX = "signin:";
21
+ export declare const PROVIDER_AUTH_SCHEMA = "\nCREATE TABLE IF NOT EXISTS provider_auth_pause (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n provider TEXT NOT NULL,\n ordinal INTEGER NOT NULL,\n auth_mode TEXT NOT NULL DEFAULT 'subscription' CHECK (auth_mode IN ('subscription', 'api-key')),\n opened_at TEXT NOT NULL,\n first_run INTEGER,\n task_ref INTEGER,\n runs INTEGER NOT NULL DEFAULT 1,\n -- 1 once a sign-in probe during this pause said the provider is signed out:\n -- only a probe that then sees it signed in again may lift the pause.\n probe_saw_out INTEGER NOT NULL DEFAULT 0,\n lifted_at TEXT,\n lifted_by TEXT,\n lifted_how TEXT CHECK (lifted_how IS NULL OR lifted_how IN ('run', 'probe', 'person')),\n resumed INTEGER,\n -- The last task let through as a trial while paused: a sign-in check can\n -- say \"logged in\" for a session that no longer works, so a real run decides.\n last_trial_at TEXT,\n UNIQUE (provider, ordinal)\n);\nCREATE UNIQUE INDEX IF NOT EXISTS provider_auth_pause_open ON provider_auth_pause (provider) WHERE lifted_at IS NULL;\n";
22
+ export type AuthPause = {
23
+ id: number;
24
+ provider: ProviderId;
25
+ ordinal: number;
26
+ authMode: "subscription" | "api-key";
27
+ openedAt: string;
28
+ firstRun: number | null;
29
+ taskRef: number | null;
30
+ runs: number;
31
+ };
32
+ export declare function providerName(provider: string): string;
33
+ /** The plain reason: "Claude needs you to sign in again". */
34
+ export declare function signInReason(pause: Pick<AuthPause, "provider" | "authMode">): string;
35
+ /** What to run, as a command. */
36
+ export declare function signInCommand(pause: Pick<AuthPause, "provider" | "authMode">): string;
37
+ /** One line for status, ready and the console. */
38
+ export declare function signInWords(pause: AuthPause): string;
39
+ /** The open pause for a provider, or null. */
40
+ export declare function authPauseOf(store: Store, provider: string): AuthPause | null;
41
+ /** The open pause the dispatch gate last left this task waiting on, or null. */
42
+ export declare function authWaitOf(store: Store, taskRef: number): AuthPause | null;
43
+ /** Every open pause, oldest first. */
44
+ export declare function openAuthPauses(store: Store): AuthPause[];
45
+ /**
46
+ * Record an auth-expired run against its provider's pause, opening the
47
+ * incident (and its one notification) when none is open. Called inside the
48
+ * run's own failure transaction. Returns whether this run opened it.
49
+ */
50
+ export declare function pauseForAuth(store: Store, args: {
51
+ provider: string;
52
+ authMode: "subscription" | "api-key" | null;
53
+ runId: number;
54
+ taskRef: number;
55
+ now: Date;
56
+ }): {
57
+ opened: boolean;
58
+ pause: AuthPause;
59
+ };
60
+ /**
61
+ * Lift a provider's open pause — a run or probe on it succeeded, or a person
62
+ * resumed it — and send the one short "signed in again" message. Returns the
63
+ * number of tasks its incident sent back to the queue, or null when nothing
64
+ * was paused.
65
+ */
66
+ export declare function liftAuthPause(store: Store, provider: string, how: "run" | "probe" | "person", by: string, now: Date,
67
+ /** For a run: when it started. A run that began before the pause opened proves nothing about the sign-in now. */
68
+ runStartedAt?: Date): {
69
+ resumed: number;
70
+ pause: AuthPause;
71
+ } | null;
72
+ /**
73
+ * What a non-spending sign-in probe said. A probe lifts the pause only after
74
+ * one during the same pause saw the provider signed out: a CLI that reports
75
+ * "logged in" while its session cannot refresh must not lift and re-open the
76
+ * pause over and over.
77
+ */
78
+ export declare function noteSignInProbe(store: Store, provider: string, state: string, now: Date): {
79
+ resumed: number;
80
+ } | null;
81
+ /** How often a paused provider lets one task through as a trial. */
82
+ export declare const AUTH_TRIAL_MS: number;
83
+ /**
84
+ * Whether this task may go ahead as the paused provider's trial: at most one
85
+ * every AUTH_TRIAL_MS. A trial that works lifts the pause (its run proves the
86
+ * sign-in); one that fails the same way only counts against the open
87
+ * incident, so nobody is told twice.
88
+ */
89
+ export declare function claimAuthTrial(store: Store, pause: AuthPause, now: Date): boolean;
90
+ /**
91
+ * The sign-in gate every road that starts an agent asks before it claims
92
+ * anything: the tick's queue, and the roads beside it (fallback entries,
93
+ * attended continuations, contest resumes). The open pause the work waits
94
+ * on, or null when it may go ahead — nothing paused, or this is the pause's
95
+ * one trial. A waiting task is noted on its task_ref, so the work index says
96
+ * exactly what the gate decided; a task let through is un-noted.
97
+ */
98
+ export declare function signInGate(store: Store, providers: readonly string[], now: Date, taskRef?: number): AuthPause | null;
99
+ /** The loop beside the console: every two minutes, probe each paused
100
+ * provider's sign-in (no model, nothing spent) and lift the pause when it
101
+ * works again. Nothing is probed while nothing is paused. */
102
+ export declare function startSignInProbes(store: Store, check: (provider: ProviderId, fresh?: boolean) => Promise<{
103
+ state: string;
104
+ }>, everyMs?: number): () => void;
105
+ /** The console's view of every open pause: title, command, one action. */
106
+ export declare function signInNotices(store: Store): {
107
+ provider: string;
108
+ title: string;
109
+ command: string;
110
+ detail: string;
111
+ resumeLabel: string;
112
+ resumeHref: string;
113
+ }[];