talon-agent 5.26.6 → 5.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/app.ts +18 -3
- package/src/backend/claude-sdk/factory.ts +2 -0
- package/src/backend/claude-sdk/one-shot.ts +22 -1
- package/src/backend/codex/factory.ts +2 -0
- package/src/backend/codex/one-shot.ts +68 -19
- package/src/backend/runtime/one-shot-hooks.ts +24 -0
- package/src/core/agent-runtime/capabilities.ts +7 -0
- package/src/core/agents/index.ts +2 -0
- package/src/core/agents/prompt.ts +75 -0
- package/src/core/agents/registry.ts +317 -7
- package/src/core/agents/runner.ts +331 -10
- package/src/core/types.ts +15 -0
- package/src/storage/agents/repo.ts +200 -0
- package/src/storage/sql/agents.sql +43 -0
- package/src/storage/sql/schema.sql +54 -0
- package/src/storage/sql/statements.generated.ts +87 -1
|
@@ -24,8 +24,19 @@
|
|
|
24
24
|
* `spawnAgent` returns as soon as the run is under way: the caller (a chat
|
|
25
25
|
* turn or another agent) keeps working and hears back through the wake turn
|
|
26
26
|
* or its mailbox.
|
|
27
|
+
*
|
|
28
|
+
* - **Restarts are not deaths.** Every agent is mirrored to the `agents`
|
|
29
|
+
* table (see the registry's `persist` hook). A daemon shutdown parks the
|
|
30
|
+
* live ones (`interruptAgentsForRestart`) before it tears the backends
|
|
31
|
+
* down, so the abort that follows is not recorded as a kill and the
|
|
32
|
+
* parent is not told the agent died. On the next boot
|
|
33
|
+
* `resumeAgentsAfterRestart` brings each one back under its original id:
|
|
34
|
+
* on a backend that can resume (Claude SDK session, Codex thread) it
|
|
35
|
+
* continues its own conversation with a short "you were interrupted"
|
|
36
|
+
* note; elsewhere it is re-briefed with the tail of its previous run log.
|
|
27
37
|
*/
|
|
28
38
|
|
|
39
|
+
import { readFile } from "node:fs/promises";
|
|
29
40
|
import { dirs } from "../../util/paths.js";
|
|
30
41
|
import { log, logError, logWarn } from "../../util/log.js";
|
|
31
42
|
import {
|
|
@@ -61,9 +72,14 @@ import {
|
|
|
61
72
|
import {
|
|
62
73
|
agentLogHeader,
|
|
63
74
|
agentLogPath,
|
|
75
|
+
agentResumeLogHeader,
|
|
64
76
|
buildAgentPrompt,
|
|
65
77
|
buildAgentSystemPrompt,
|
|
78
|
+
buildRebriefPrompt,
|
|
79
|
+
buildResumePrompt,
|
|
66
80
|
} from "./prompt.js";
|
|
81
|
+
import * as agentsRepo from "../../storage/agents/repo.js";
|
|
82
|
+
import type { PersistedAgent } from "../../storage/agents/repo.js";
|
|
67
83
|
import { agentRegistry } from "./registry.js";
|
|
68
84
|
import type {
|
|
69
85
|
AgentCaps,
|
|
@@ -259,6 +275,10 @@ export async function spawnAgent(
|
|
|
259
275
|
...(spec.reasoningEffort
|
|
260
276
|
? { reasoningEffort: spec.reasoningEffort }
|
|
261
277
|
: {}),
|
|
278
|
+
...(spec.model ? { requestedModel: spec.model } : {}),
|
|
279
|
+
timeoutMs: spec.timeoutMs ?? capsHolder.caps.defaultTimeoutMs,
|
|
280
|
+
cwd: dirs.workspace,
|
|
281
|
+
...(spec.preflight ? { preflight: true } : {}),
|
|
262
282
|
},
|
|
263
283
|
capsHolder.caps,
|
|
264
284
|
);
|
|
@@ -295,6 +315,16 @@ export async function spawnAgent(
|
|
|
295
315
|
};
|
|
296
316
|
}
|
|
297
317
|
|
|
318
|
+
/**
|
|
319
|
+
* How a restarted run picks up. `sessionId` set = continue that backend
|
|
320
|
+
* conversation; unset = a fresh conversation re-briefed with `prompt`.
|
|
321
|
+
*/
|
|
322
|
+
interface ResumePlan {
|
|
323
|
+
readonly prompt: string;
|
|
324
|
+
readonly sessionId?: string;
|
|
325
|
+
readonly interruptedAt: number;
|
|
326
|
+
}
|
|
327
|
+
|
|
298
328
|
/** Build the one-shot params for a run, wired to its log and text capture. */
|
|
299
329
|
async function buildRunParams(
|
|
300
330
|
record: AgentRecord,
|
|
@@ -302,15 +332,26 @@ async function buildRunParams(
|
|
|
302
332
|
model: string,
|
|
303
333
|
abortController: AbortController,
|
|
304
334
|
capture: { last: string },
|
|
335
|
+
resume?: ResumePlan,
|
|
305
336
|
): Promise<OneShotAgentParams> {
|
|
306
337
|
const appendLog = await openRunLog(
|
|
307
338
|
agentLogPath(record.id),
|
|
308
|
-
|
|
339
|
+
resume
|
|
340
|
+
? agentResumeLogHeader(
|
|
341
|
+
record,
|
|
342
|
+
model,
|
|
343
|
+
resume.interruptedAt,
|
|
344
|
+
resume.sessionId,
|
|
345
|
+
)
|
|
346
|
+
: agentLogHeader(record, model),
|
|
309
347
|
);
|
|
348
|
+
const id = record.id;
|
|
310
349
|
return {
|
|
311
|
-
prompt:
|
|
312
|
-
|
|
313
|
-
|
|
350
|
+
prompt: resume
|
|
351
|
+
? resume.prompt
|
|
352
|
+
: buildAgentPrompt(record.brief, {
|
|
353
|
+
preflight: spec.preflight === true,
|
|
354
|
+
}),
|
|
314
355
|
systemPrompt: buildAgentSystemPrompt({
|
|
315
356
|
agentId: record.id,
|
|
316
357
|
label: record.label,
|
|
@@ -327,6 +368,10 @@ async function buildRunParams(
|
|
|
327
368
|
const trimmed = text.trim();
|
|
328
369
|
if (trimmed) capture.last = trimmed;
|
|
329
370
|
},
|
|
371
|
+
// Persisted the moment the backend reports it, so a restart at any
|
|
372
|
+
// point after the first message can resume the conversation.
|
|
373
|
+
onSessionId: (sessionId) => agentRegistry.setSessionId(id, sessionId),
|
|
374
|
+
...(resume?.sessionId ? { resumeSessionId: resume.sessionId } : {}),
|
|
330
375
|
...(spec.reasoningEffort ? { reasoningEffort: spec.reasoningEffort } : {}),
|
|
331
376
|
};
|
|
332
377
|
}
|
|
@@ -389,6 +434,7 @@ async function runAgent(
|
|
|
389
434
|
spec: AgentSpawnSpec,
|
|
390
435
|
resolved: { model: string; background: BackgroundRunner },
|
|
391
436
|
acquired: Awaited<ReturnType<typeof acquireBackendInstance>>,
|
|
437
|
+
resume?: ResumePlan,
|
|
392
438
|
): Promise<void> {
|
|
393
439
|
const { model, background } = resolved;
|
|
394
440
|
const { release } = acquired;
|
|
@@ -419,8 +465,11 @@ async function runAgent(
|
|
|
419
465
|
model,
|
|
420
466
|
abortController,
|
|
421
467
|
capture,
|
|
468
|
+
resume,
|
|
422
469
|
);
|
|
423
|
-
if (
|
|
470
|
+
if (agentRegistry.isInterrupted(id)) {
|
|
471
|
+
settled = null;
|
|
472
|
+
} else if (abortController.signal.aborted) {
|
|
424
473
|
// A kill that lands during startup — while the backend is being
|
|
425
474
|
// acquired or the log opened — must not be lost. Handing an
|
|
426
475
|
// already-aborted signal to a backend relies on it checking, and not
|
|
@@ -441,18 +490,38 @@ async function runAgent(
|
|
|
441
490
|
evictLabel: agentContextLabel(id),
|
|
442
491
|
});
|
|
443
492
|
recordBackendRunUsage(record.backendId, usage ?? undefined);
|
|
444
|
-
|
|
445
|
-
|
|
493
|
+
// A backend may swallow the shutdown abort and return normally — the
|
|
494
|
+
// run still did not finish, so it must not settle as done/failed, and
|
|
495
|
+
// it says nothing about the backend's health either way.
|
|
496
|
+
if (agentRegistry.isInterrupted(id)) {
|
|
497
|
+
settled = null;
|
|
498
|
+
} else {
|
|
499
|
+
recordBackendRunSuccess(record.backendId);
|
|
500
|
+
settled = settleSuccess(id, task, capture.last, usage ?? undefined);
|
|
501
|
+
}
|
|
446
502
|
}
|
|
447
503
|
} catch (err) {
|
|
448
|
-
|
|
449
|
-
|
|
504
|
+
if (agentRegistry.isInterrupted(id)) {
|
|
505
|
+
settled = null;
|
|
506
|
+
} else {
|
|
507
|
+
recordBackendRunFailure(record.backendId, err);
|
|
508
|
+
settled = settleFailure(id, task, err);
|
|
509
|
+
}
|
|
450
510
|
} finally {
|
|
451
511
|
await release().catch((err: unknown) =>
|
|
452
512
|
logError("agents", `failed to release backend for ${id}`, err),
|
|
453
513
|
);
|
|
454
514
|
}
|
|
455
515
|
|
|
516
|
+
if (agentRegistry.isInterrupted(id)) {
|
|
517
|
+
// Parked by a daemon shutdown: the persisted row stays `running` for
|
|
518
|
+
// the next boot to resume. No settlement, no delivery, no reaping — the
|
|
519
|
+
// parent is not told its agent died, because it didn't.
|
|
520
|
+
task.fail(new Error("interrupted by daemon shutdown — will resume"));
|
|
521
|
+
agentRegistry.releaseInterrupted(id);
|
|
522
|
+
log("agents", `${id} "${record.label}" interrupted by shutdown — parked`);
|
|
523
|
+
return;
|
|
524
|
+
}
|
|
456
525
|
if (!settled) return;
|
|
457
526
|
log(
|
|
458
527
|
"agents",
|
|
@@ -491,12 +560,264 @@ export function killAgent(agentId: string): boolean {
|
|
|
491
560
|
return agentRegistry.requestKill(agentId);
|
|
492
561
|
}
|
|
493
562
|
|
|
563
|
+
/**
|
|
564
|
+
* Park every live agent for the next boot. Called FIRST in a graceful
|
|
565
|
+
* shutdown — before the frontends and the backend pool go down, since
|
|
566
|
+
* tearing a backend down aborts the runs on it and an unparked agent would
|
|
567
|
+
* record that abort as its death. Idempotent.
|
|
568
|
+
*/
|
|
569
|
+
export function interruptAgentsForRestart(): number {
|
|
570
|
+
const parked = agentRegistry.interruptAll();
|
|
571
|
+
if (parked > 0) {
|
|
572
|
+
log("agents", `Shutdown: parked ${parked} running agent(s) for resume`);
|
|
573
|
+
}
|
|
574
|
+
return parked;
|
|
575
|
+
}
|
|
576
|
+
|
|
494
577
|
/**
|
|
495
578
|
* Abort every live agent — the shutdown lever, alongside heartbeat's and
|
|
496
|
-
* cron's.
|
|
579
|
+
* cron's. Agents are parked first (a no-op if the shutdown already did), so
|
|
580
|
+
* this abort ends the process's hold on them without ending the agents:
|
|
581
|
+
* the next boot resumes them. Returns how many aborts were requested.
|
|
497
582
|
*/
|
|
498
583
|
export function shutdownAgents(): number {
|
|
584
|
+
interruptAgentsForRestart();
|
|
499
585
|
const killed = agentRegistry.killAll();
|
|
500
586
|
if (killed > 0) log("agents", `Shutdown: aborted ${killed} running agent(s)`);
|
|
501
587
|
return killed;
|
|
502
588
|
}
|
|
589
|
+
|
|
590
|
+
// ── Resume after restart ─────────────────────────────────────────────────────
|
|
591
|
+
|
|
592
|
+
/** A restart may resume one agent at most this many times. */
|
|
593
|
+
export const MAX_AGENT_RESUMES = 3;
|
|
594
|
+
/** An agent interrupted longer ago than this is not resumed. */
|
|
595
|
+
const AGENT_RESUME_STALE_MS = 24 * 60 * 60 * 1000;
|
|
596
|
+
/** A resumed run always gets at least this much wall-clock. */
|
|
597
|
+
const AGENT_RESUME_MIN_TIMEOUT_MS = 10 * 60 * 1000;
|
|
598
|
+
/** Settled rows are kept this long for inspection, then pruned at boot. */
|
|
599
|
+
const SETTLED_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
|
|
600
|
+
/** How much of the previous run log a re-briefed agent is shown. */
|
|
601
|
+
const REBRIEF_LOG_TAIL_CHARS = 12_000;
|
|
602
|
+
|
|
603
|
+
/** The last `max` characters of an agent's previous run log, if any. */
|
|
604
|
+
async function previousLogTail(agentId: string, max: number): Promise<string> {
|
|
605
|
+
try {
|
|
606
|
+
const text = await readFile(agentLogPath(agentId), "utf-8");
|
|
607
|
+
return text.length > max ? `…${text.slice(text.length - max)}` : text;
|
|
608
|
+
} catch {
|
|
609
|
+
return "";
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
/** Settle a restored agent without running it, and tell its parent. */
|
|
614
|
+
async function settleRestored(
|
|
615
|
+
record: AgentRecord,
|
|
616
|
+
patch: Parameters<typeof agentRegistry.settle>[1],
|
|
617
|
+
): Promise<void> {
|
|
618
|
+
const settled = agentRegistry.settle(record.id, patch);
|
|
619
|
+
if (!settled) return;
|
|
620
|
+
log(
|
|
621
|
+
"agents",
|
|
622
|
+
`${record.id} "${record.label}" → ${settled.state} (after restart)`,
|
|
623
|
+
);
|
|
624
|
+
reapChildren(settled);
|
|
625
|
+
await deliverSettlement(settled).catch((err: unknown) =>
|
|
626
|
+
logError("agents", `delivery failed for ${record.id}`, err),
|
|
627
|
+
);
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
/** Mark a row that cannot even be restored (its parent agent is gone). */
|
|
631
|
+
function abandonRow(saved: PersistedAgent, error: string): void {
|
|
632
|
+
try {
|
|
633
|
+
agentsRepo.upsert({
|
|
634
|
+
...saved,
|
|
635
|
+
state: "failed",
|
|
636
|
+
error,
|
|
637
|
+
endedAt: Date.now(),
|
|
638
|
+
updatedAt: Date.now(),
|
|
639
|
+
});
|
|
640
|
+
} catch (err) {
|
|
641
|
+
logError("agents", `could not mark ${saved.id} abandoned`, err);
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
/**
|
|
646
|
+
* Resolve the model a resumed run uses: the one it ran on, else the one it
|
|
647
|
+
* asked for, else the backend default — a model withdrawn across the
|
|
648
|
+
* restart must not strand the agent.
|
|
649
|
+
*/
|
|
650
|
+
async function resolveResumeRun(
|
|
651
|
+
backend: Backend,
|
|
652
|
+
backendId: string,
|
|
653
|
+
candidates: ReadonlyArray<string | undefined>,
|
|
654
|
+
): Promise<Awaited<ReturnType<typeof resolveRun>>> {
|
|
655
|
+
let last: Awaited<ReturnType<typeof resolveRun>> = {
|
|
656
|
+
ok: false,
|
|
657
|
+
error: "no model",
|
|
658
|
+
};
|
|
659
|
+
const tried = new Set<string | undefined>();
|
|
660
|
+
for (const candidate of [...candidates, undefined]) {
|
|
661
|
+
if (tried.has(candidate)) continue;
|
|
662
|
+
tried.add(candidate);
|
|
663
|
+
last = await resolveRun(backend, backendId, candidate);
|
|
664
|
+
if (last.ok) return last;
|
|
665
|
+
}
|
|
666
|
+
return last;
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
/** Bring one interrupted agent back. Never throws. */
|
|
670
|
+
async function resumeOne(saved: PersistedAgent, now: number): Promise<void> {
|
|
671
|
+
// A crash leaves no interruption stamp: charge the run up to its last
|
|
672
|
+
// persisted write, which is as close as anyone can know.
|
|
673
|
+
const interruptedAt = saved.interruptedAt ?? saved.updatedAt;
|
|
674
|
+
const elapsedMs =
|
|
675
|
+
saved.interruptedAt === undefined && saved.startedAt !== undefined
|
|
676
|
+
? saved.elapsedMs + Math.max(0, saved.updatedAt - saved.startedAt)
|
|
677
|
+
: saved.elapsedMs;
|
|
678
|
+
|
|
679
|
+
// Already back (a second resume pass in the same process) — leave it be.
|
|
680
|
+
if (agentRegistry.isLive(saved.id)) return;
|
|
681
|
+
const record = agentRegistry.restore({ ...saved, elapsedMs, interruptedAt });
|
|
682
|
+
if (!record) {
|
|
683
|
+
abandonRow(
|
|
684
|
+
saved,
|
|
685
|
+
"interrupted by a daemon restart; its parent agent did not survive it",
|
|
686
|
+
);
|
|
687
|
+
logWarn("agents", `${saved.id}: parent gone after restart — abandoned`);
|
|
688
|
+
return;
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
if (saved.reported) {
|
|
692
|
+
// It finished its job (report_result landed) and was only waiting to
|
|
693
|
+
// wind down — deliver what it said instead of running it again.
|
|
694
|
+
await settleRestored(record, { state: "done" });
|
|
695
|
+
return;
|
|
696
|
+
}
|
|
697
|
+
if (saved.resumeCount >= MAX_AGENT_RESUMES) {
|
|
698
|
+
await settleRestored(record, {
|
|
699
|
+
state: "failed",
|
|
700
|
+
error:
|
|
701
|
+
`interrupted by ${saved.resumeCount + 1} daemon restarts; not ` +
|
|
702
|
+
`resumed again. Its run log is ${agentLogPath(saved.id)}.`,
|
|
703
|
+
});
|
|
704
|
+
return;
|
|
705
|
+
}
|
|
706
|
+
if (now - interruptedAt > AGENT_RESUME_STALE_MS) {
|
|
707
|
+
await settleRestored(record, {
|
|
708
|
+
state: "failed",
|
|
709
|
+
error:
|
|
710
|
+
`interrupted by a daemon restart at ${new Date(interruptedAt).toISOString()} ` +
|
|
711
|
+
`and the daemon was down too long to resume it. Its run log is ` +
|
|
712
|
+
`${agentLogPath(saved.id)}.`,
|
|
713
|
+
});
|
|
714
|
+
return;
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
const backendId = saved.backendId;
|
|
718
|
+
let acquired: Awaited<ReturnType<typeof acquireBackendInstance>>;
|
|
719
|
+
try {
|
|
720
|
+
acquired = await acquireBackendInstance(backendId);
|
|
721
|
+
} catch (err) {
|
|
722
|
+
await settleRestored(record, {
|
|
723
|
+
state: "failed",
|
|
724
|
+
error:
|
|
725
|
+
`interrupted by a daemon restart, and its backend "${backendId}" ` +
|
|
726
|
+
`is unavailable after it: ${errText(err)}`,
|
|
727
|
+
});
|
|
728
|
+
return;
|
|
729
|
+
}
|
|
730
|
+
const resolved = await resolveResumeRun(acquired.backend, backendId, [
|
|
731
|
+
saved.model,
|
|
732
|
+
saved.requestedModel,
|
|
733
|
+
]);
|
|
734
|
+
if (!resolved.ok) {
|
|
735
|
+
await acquired.release().catch(() => {});
|
|
736
|
+
await settleRestored(record, {
|
|
737
|
+
state: "failed",
|
|
738
|
+
error: `interrupted by a daemon restart and could not resume: ${resolved.error}`,
|
|
739
|
+
});
|
|
740
|
+
return;
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
const canResume =
|
|
744
|
+
resolved.background.supportsResume === true &&
|
|
745
|
+
saved.sessionId !== undefined;
|
|
746
|
+
const minutes = Math.round(elapsedMs / 60_000);
|
|
747
|
+
const plan: ResumePlan = canResume
|
|
748
|
+
? {
|
|
749
|
+
prompt: buildResumePrompt({ interruptedAt, elapsedMinutes: minutes }),
|
|
750
|
+
sessionId: saved.sessionId!,
|
|
751
|
+
interruptedAt,
|
|
752
|
+
}
|
|
753
|
+
: {
|
|
754
|
+
prompt: buildRebriefPrompt({
|
|
755
|
+
brief: saved.brief,
|
|
756
|
+
preflight: saved.preflight === true,
|
|
757
|
+
interruptedAt,
|
|
758
|
+
elapsedMinutes: minutes,
|
|
759
|
+
logPath: agentLogPath(saved.id),
|
|
760
|
+
logTail: await previousLogTail(saved.id, REBRIEF_LOG_TAIL_CHARS),
|
|
761
|
+
}),
|
|
762
|
+
interruptedAt,
|
|
763
|
+
};
|
|
764
|
+
|
|
765
|
+
const budget =
|
|
766
|
+
(saved.timeoutMs ?? capsHolder.caps.defaultTimeoutMs) - elapsedMs;
|
|
767
|
+
const timeoutMs = Math.max(AGENT_RESUME_MIN_TIMEOUT_MS, budget);
|
|
768
|
+
const spec: AgentSpawnSpec = {
|
|
769
|
+
brief: saved.brief,
|
|
770
|
+
label: saved.label,
|
|
771
|
+
parent: record.parent,
|
|
772
|
+
backendId,
|
|
773
|
+
...(saved.requestedModel ? { model: saved.requestedModel } : {}),
|
|
774
|
+
...(record.reasoningEffort
|
|
775
|
+
? { reasoningEffort: record.reasoningEffort }
|
|
776
|
+
: {}),
|
|
777
|
+
timeoutMs,
|
|
778
|
+
...(saved.preflight ? { preflight: true } : {}),
|
|
779
|
+
};
|
|
780
|
+
agentRegistry.markResumed(record.id);
|
|
781
|
+
log(
|
|
782
|
+
"agents",
|
|
783
|
+
`${record.id} "${record.label}" resuming after restart ` +
|
|
784
|
+
`(${canResume ? `session ${saved.sessionId}` : "re-briefed"}, ` +
|
|
785
|
+
`${backendId}/${resolved.model}, ${Math.round(timeoutMs / 1000)}s left, ` +
|
|
786
|
+
`resume #${saved.resumeCount + 1})`,
|
|
787
|
+
);
|
|
788
|
+
void runAgent(record, spec, resolved, acquired, plan);
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
/**
|
|
792
|
+
* Boot-time: respawn every agent the previous process left running. Call
|
|
793
|
+
* once, after the dispatcher, backend pool and frontends are up (a resumed
|
|
794
|
+
* agent reaches its tools and its parent through them). Parents come back
|
|
795
|
+
* before their children, so a child re-attaches to its live parent.
|
|
796
|
+
* Returns how many rows were considered.
|
|
797
|
+
*/
|
|
798
|
+
export async function resumeAgentsAfterRestart(): Promise<number> {
|
|
799
|
+
const now = Date.now();
|
|
800
|
+
try {
|
|
801
|
+
const pruned = agentsRepo.pruneSettled(now - SETTLED_RETENTION_MS);
|
|
802
|
+
if (pruned > 0) log("agents", `Pruned ${pruned} settled agent row(s)`);
|
|
803
|
+
} catch (err) {
|
|
804
|
+
logError("agents", "prune of settled agent rows failed", err);
|
|
805
|
+
}
|
|
806
|
+
let rows: PersistedAgent[];
|
|
807
|
+
try {
|
|
808
|
+
rows = agentsRepo.listInterrupted();
|
|
809
|
+
} catch (err) {
|
|
810
|
+
logError("agents", "could not read interrupted agents", err);
|
|
811
|
+
return 0;
|
|
812
|
+
}
|
|
813
|
+
if (rows.length === 0) return 0;
|
|
814
|
+
log("agents", `Resuming ${rows.length} agent(s) interrupted by the restart`);
|
|
815
|
+
for (const saved of rows) {
|
|
816
|
+
try {
|
|
817
|
+
await resumeOne(saved, now);
|
|
818
|
+
} catch (err) {
|
|
819
|
+
logError("agents", `resume of ${saved.id} failed`, err);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
return rows.length;
|
|
823
|
+
}
|
package/src/core/types.ts
CHANGED
|
@@ -172,6 +172,21 @@ export type OneShotAgentParams = {
|
|
|
172
172
|
* and logs a throwing callback so a consumer bug can never fail the run.
|
|
173
173
|
*/
|
|
174
174
|
onAssistantText?: (text: string) => void;
|
|
175
|
+
/**
|
|
176
|
+
* Resume this backend conversation instead of starting a fresh one — the
|
|
177
|
+
* handle an earlier run reported through `onSessionId`. Honoured only by
|
|
178
|
+
* backends whose `BackgroundRunner.supportsResume` is true (Claude SDK:
|
|
179
|
+
* session id; Codex: thread id); the others ignore it and start fresh,
|
|
180
|
+
* which is why the sub-agent runner checks that flag before relying on it.
|
|
181
|
+
*/
|
|
182
|
+
resumeSessionId?: string;
|
|
183
|
+
/**
|
|
184
|
+
* Called once the backend knows the run's conversation handle (Claude SDK
|
|
185
|
+
* session id, Codex thread id) — a sub-agent persists it so a daemon
|
|
186
|
+
* restart can resume the conversation. Best-effort and synchronous, like
|
|
187
|
+
* `onAssistantText`: a throwing callback must never fail the run.
|
|
188
|
+
*/
|
|
189
|
+
onSessionId?: (sessionId: string) => void;
|
|
175
190
|
};
|
|
176
191
|
|
|
177
192
|
/** How much cache telemetry a backend can surface in /status. */
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Agents repository — executes the statements in sql/agents.sql against
|
|
3
|
+
* the `agents` table; no SQL text lives here. The domain side (what gets
|
|
4
|
+
* written when, and the boot-time resume) is core/agents/registry.ts and
|
|
5
|
+
* core/agents/runner.ts.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { getDatabase } from "../db.js";
|
|
9
|
+
import { agentsSql } from "../sql/statements.generated.js";
|
|
10
|
+
|
|
11
|
+
/** One persisted message waiting in an agent's mailbox. */
|
|
12
|
+
type PersistedAgentMessage = {
|
|
13
|
+
from: string;
|
|
14
|
+
text: string;
|
|
15
|
+
at: number;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
/** A sub-agent as stored — everything a restart needs to bring it back. */
|
|
19
|
+
export type PersistedAgent = {
|
|
20
|
+
id: string;
|
|
21
|
+
label: string;
|
|
22
|
+
brief: string;
|
|
23
|
+
parentKind: "chat" | "agent";
|
|
24
|
+
/** Chat key (chat parent) or agent id (agent parent). */
|
|
25
|
+
parentId: string;
|
|
26
|
+
/** Frontend numeric chat id — chat parents only. */
|
|
27
|
+
parentNumericChatId?: number;
|
|
28
|
+
backendId: string;
|
|
29
|
+
model?: string;
|
|
30
|
+
requestedModel?: string;
|
|
31
|
+
reasoningEffort?: string;
|
|
32
|
+
timeoutMs?: number;
|
|
33
|
+
depth: number;
|
|
34
|
+
cwd?: string;
|
|
35
|
+
state: string;
|
|
36
|
+
createdAt: number;
|
|
37
|
+
startedAt?: number;
|
|
38
|
+
endedAt?: number;
|
|
39
|
+
updatedAt: number;
|
|
40
|
+
sessionId?: string;
|
|
41
|
+
inbox: PersistedAgentMessage[];
|
|
42
|
+
reported: boolean;
|
|
43
|
+
resultSummary?: string;
|
|
44
|
+
resultDetails?: string;
|
|
45
|
+
error?: string;
|
|
46
|
+
elapsedMs: number;
|
|
47
|
+
resumeCount: number;
|
|
48
|
+
interruptedAt?: number;
|
|
49
|
+
/** The spawn asked for the pre-flight lane instruction. */
|
|
50
|
+
preflight?: boolean;
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
type Row = {
|
|
54
|
+
id: string;
|
|
55
|
+
label: string;
|
|
56
|
+
brief: string;
|
|
57
|
+
parent_kind: string;
|
|
58
|
+
parent_id: string;
|
|
59
|
+
parent_numeric: number | null;
|
|
60
|
+
backend_id: string;
|
|
61
|
+
model: string | null;
|
|
62
|
+
requested_model: string | null;
|
|
63
|
+
effort: string | null;
|
|
64
|
+
timeout_ms: number | null;
|
|
65
|
+
depth: number;
|
|
66
|
+
cwd: string | null;
|
|
67
|
+
state: string;
|
|
68
|
+
created_at: number;
|
|
69
|
+
started_at: number | null;
|
|
70
|
+
ended_at: number | null;
|
|
71
|
+
updated_at: number;
|
|
72
|
+
session_id: string | null;
|
|
73
|
+
inbox_json: string;
|
|
74
|
+
reported: number;
|
|
75
|
+
result_summary: string | null;
|
|
76
|
+
result_details: string | null;
|
|
77
|
+
error: string | null;
|
|
78
|
+
elapsed_ms: number;
|
|
79
|
+
resume_count: number;
|
|
80
|
+
interrupted_at: number | null;
|
|
81
|
+
preflight: number;
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
function parseInbox(raw: string): PersistedAgentMessage[] {
|
|
85
|
+
try {
|
|
86
|
+
const parsed: unknown = JSON.parse(raw);
|
|
87
|
+
if (!Array.isArray(parsed)) return [];
|
|
88
|
+
return parsed.filter(
|
|
89
|
+
(m): m is PersistedAgentMessage =>
|
|
90
|
+
typeof m === "object" &&
|
|
91
|
+
m !== null &&
|
|
92
|
+
typeof (m as { from?: unknown }).from === "string" &&
|
|
93
|
+
typeof (m as { text?: unknown }).text === "string" &&
|
|
94
|
+
typeof (m as { at?: unknown }).at === "number",
|
|
95
|
+
);
|
|
96
|
+
} catch {
|
|
97
|
+
return [];
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function rowToAgent(row: Row): PersistedAgent {
|
|
102
|
+
const agent: PersistedAgent = {
|
|
103
|
+
id: row.id,
|
|
104
|
+
label: row.label,
|
|
105
|
+
brief: row.brief,
|
|
106
|
+
parentKind: row.parent_kind === "agent" ? "agent" : "chat",
|
|
107
|
+
parentId: row.parent_id,
|
|
108
|
+
backendId: row.backend_id,
|
|
109
|
+
depth: row.depth,
|
|
110
|
+
state: row.state,
|
|
111
|
+
createdAt: row.created_at,
|
|
112
|
+
updatedAt: row.updated_at,
|
|
113
|
+
inbox: parseInbox(row.inbox_json),
|
|
114
|
+
reported: row.reported === 1,
|
|
115
|
+
elapsedMs: row.elapsed_ms,
|
|
116
|
+
resumeCount: row.resume_count,
|
|
117
|
+
};
|
|
118
|
+
if (row.parent_numeric !== null)
|
|
119
|
+
agent.parentNumericChatId = row.parent_numeric;
|
|
120
|
+
if (row.model !== null) agent.model = row.model;
|
|
121
|
+
if (row.requested_model !== null) agent.requestedModel = row.requested_model;
|
|
122
|
+
if (row.effort !== null) agent.reasoningEffort = row.effort;
|
|
123
|
+
if (row.timeout_ms !== null) agent.timeoutMs = row.timeout_ms;
|
|
124
|
+
if (row.cwd !== null) agent.cwd = row.cwd;
|
|
125
|
+
if (row.started_at !== null) agent.startedAt = row.started_at;
|
|
126
|
+
if (row.ended_at !== null) agent.endedAt = row.ended_at;
|
|
127
|
+
if (row.session_id !== null) agent.sessionId = row.session_id;
|
|
128
|
+
if (row.result_summary !== null) agent.resultSummary = row.result_summary;
|
|
129
|
+
if (row.result_details !== null) agent.resultDetails = row.result_details;
|
|
130
|
+
if (row.error !== null) agent.error = row.error;
|
|
131
|
+
if (row.interrupted_at !== null) agent.interruptedAt = row.interrupted_at;
|
|
132
|
+
if (row.preflight === 1) agent.preflight = true;
|
|
133
|
+
return agent;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export function upsert(a: PersistedAgent): void {
|
|
137
|
+
getDatabase()
|
|
138
|
+
.prepare(agentsSql.upsert)
|
|
139
|
+
.run(
|
|
140
|
+
a.id,
|
|
141
|
+
a.label,
|
|
142
|
+
a.brief,
|
|
143
|
+
a.parentKind,
|
|
144
|
+
a.parentId,
|
|
145
|
+
a.parentNumericChatId ?? null,
|
|
146
|
+
a.backendId,
|
|
147
|
+
a.model ?? null,
|
|
148
|
+
a.requestedModel ?? null,
|
|
149
|
+
a.reasoningEffort ?? null,
|
|
150
|
+
a.timeoutMs ?? null,
|
|
151
|
+
a.depth,
|
|
152
|
+
a.cwd ?? null,
|
|
153
|
+
a.state,
|
|
154
|
+
a.createdAt,
|
|
155
|
+
a.startedAt ?? null,
|
|
156
|
+
a.endedAt ?? null,
|
|
157
|
+
a.updatedAt,
|
|
158
|
+
a.sessionId ?? null,
|
|
159
|
+
JSON.stringify(a.inbox),
|
|
160
|
+
a.reported ? 1 : 0,
|
|
161
|
+
a.resultSummary ?? null,
|
|
162
|
+
a.resultDetails ?? null,
|
|
163
|
+
a.error ?? null,
|
|
164
|
+
a.elapsedMs,
|
|
165
|
+
a.resumeCount,
|
|
166
|
+
a.interruptedAt ?? null,
|
|
167
|
+
a.preflight ? 1 : 0,
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export function get(id: string): PersistedAgent | undefined {
|
|
172
|
+
const row = getDatabase().prepare(agentsSql.get).get(id) as Row | undefined;
|
|
173
|
+
return row ? rowToAgent(row) : undefined;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** Rows still queued/running — interrupted by the last restart. */
|
|
177
|
+
export function listInterrupted(): PersistedAgent[] {
|
|
178
|
+
const rows = getDatabase().prepare(agentsSql.listInterrupted).all() as Row[];
|
|
179
|
+
return rows.map(rowToAgent);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export function remove(id: string): boolean {
|
|
183
|
+
const result = getDatabase().prepare(agentsSql.remove).run(id) as {
|
|
184
|
+
changes?: number;
|
|
185
|
+
};
|
|
186
|
+
return (result.changes ?? 0) > 0;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Delete settled rows last touched before `cutoff`. Returns how many. */
|
|
190
|
+
export function pruneSettled(cutoff: number): number {
|
|
191
|
+
const result = getDatabase().prepare(agentsSql.pruneSettled).run(cutoff) as {
|
|
192
|
+
changes?: number;
|
|
193
|
+
};
|
|
194
|
+
return result.changes ?? 0;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Test-only: wipe the table between suites sharing a worker DB. */
|
|
198
|
+
export function removeAll(): void {
|
|
199
|
+
getDatabase().prepare(agentsSql.removeAll).run();
|
|
200
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
-- Statements for the agents table (see storage/agents/repo.ts for
|
|
2
|
+
-- the parameter order and row↔domain mapping).
|
|
3
|
+
|
|
4
|
+
-- name: upsert
|
|
5
|
+
INSERT OR REPLACE INTO agents
|
|
6
|
+
(id, label, brief, parent_kind, parent_id, parent_numeric, backend_id,
|
|
7
|
+
model, requested_model, effort, timeout_ms, depth, cwd, state,
|
|
8
|
+
created_at, started_at, ended_at, updated_at, session_id, inbox_json,
|
|
9
|
+
reported, result_summary, result_details, error, elapsed_ms,
|
|
10
|
+
resume_count, interrupted_at, preflight)
|
|
11
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
12
|
+
|
|
13
|
+
-- name: get
|
|
14
|
+
SELECT id, label, brief, parent_kind, parent_id, parent_numeric, backend_id,
|
|
15
|
+
model, requested_model, effort, timeout_ms, depth, cwd, state,
|
|
16
|
+
created_at, started_at, ended_at, updated_at, session_id, inbox_json,
|
|
17
|
+
reported, result_summary, result_details, error, elapsed_ms,
|
|
18
|
+
resume_count, interrupted_at, preflight
|
|
19
|
+
FROM agents WHERE id = ?
|
|
20
|
+
|
|
21
|
+
-- Rows a restart interrupted, parents before children (a child can only
|
|
22
|
+
-- be re-attached to a parent that is already back in the registry).
|
|
23
|
+
|
|
24
|
+
-- name: listInterrupted
|
|
25
|
+
SELECT id, label, brief, parent_kind, parent_id, parent_numeric, backend_id,
|
|
26
|
+
model, requested_model, effort, timeout_ms, depth, cwd, state,
|
|
27
|
+
created_at, started_at, ended_at, updated_at, session_id, inbox_json,
|
|
28
|
+
reported, result_summary, result_details, error, elapsed_ms,
|
|
29
|
+
resume_count, interrupted_at, preflight
|
|
30
|
+
FROM agents WHERE state IN ('queued', 'running')
|
|
31
|
+
ORDER BY depth, created_at
|
|
32
|
+
|
|
33
|
+
-- name: remove
|
|
34
|
+
DELETE FROM agents WHERE id = ?
|
|
35
|
+
|
|
36
|
+
-- Retention: settled rows past their window. Live rows are never pruned.
|
|
37
|
+
|
|
38
|
+
-- name: pruneSettled
|
|
39
|
+
DELETE FROM agents
|
|
40
|
+
WHERE state NOT IN ('queued', 'running') AND updated_at < ?
|
|
41
|
+
|
|
42
|
+
-- name: removeAll
|
|
43
|
+
DELETE FROM agents
|