pi-durable-subagents 1.0.16 → 1.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/README.md +85 -21
- package/dist/agent/main/snapshots.js +6 -2
- package/dist/agent/main/tool.js +8 -5
- package/dist/agent/main.js +18 -1
- package/dist/cli/control.js +4 -0
- package/dist/cli/hold.js +184 -0
- package/dist/cli/main.js +65 -9
- package/dist/cli/restart.js +71 -0
- package/dist/compat/spec.js +3 -1
- package/dist/kernel/mailbox.js +1 -1
- package/dist/orchestrator/config.js +4 -2
- package/dist/orchestrator/engine.js +26 -1
- package/dist/orchestrator/executor/index.js +158 -20
- package/dist/orchestrator/executor/indexes.js +63 -0
- package/dist/orchestrator/executor/observe.js +51 -20
- package/dist/orchestrator/executor/sweep.js +40 -23
- package/dist/orchestrator/executor/usage.js +22 -12
- package/dist/orchestrator/executor/worktree.js +1 -1
- package/dist/orchestrator/ledger.js +11 -2
- package/dist/orchestrator/main.js +21 -5
- package/dist/orchestrator/snapshot.js +24 -14
- package/dist/orchestrator/store.js +15 -3
- package/dist/paths.js +2 -0
- package/dist/platform/containment.js +13 -7
- package/dist/platform/lease.js +236 -0
- package/dist/platform/proctable.js +37 -10
- package/dist/ui/cards.js +35 -17
- package/dist/ui/index.js +5 -7
- package/package.json +1 -1
|
@@ -22,20 +22,30 @@ export function sessionUsage(entries, call) {
|
|
|
22
22
|
});
|
|
23
23
|
}
|
|
24
24
|
/** P31: Sum deduplicated committed message usage across executions, optionally across a whole workflow. */
|
|
25
|
+
const totals = new WeakMap();
|
|
25
26
|
export function totalUsage(entries, call) {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
27
|
+
let state = totals.get(entries);
|
|
28
|
+
if (!state) {
|
|
29
|
+
state = { count: 0, seen: new Set(), total: { input: 0, output: 0, costUsd: 0 }, calls: new Map() };
|
|
30
|
+
totals.set(entries, state);
|
|
31
|
+
}
|
|
32
|
+
while (state.count < entries.length) {
|
|
33
|
+
const e = entries[state.count++];
|
|
34
|
+
if (e.type !== "usage")
|
|
35
|
+
continue;
|
|
36
|
+
const id = `${e.call}:${e.id}`;
|
|
37
|
+
if (state.seen.has(id))
|
|
38
|
+
continue;
|
|
39
|
+
state.seen.add(id);
|
|
40
|
+
const key = String(e.call), total = state.calls.get(key) ?? { input: 0, output: 0, costUsd: 0 }, u = e.usage;
|
|
41
|
+
state.calls.set(key, total);
|
|
42
|
+
for (const target of [total, state.total]) {
|
|
43
|
+
target.input += u.input;
|
|
44
|
+
target.output += u.output;
|
|
45
|
+
target.costUsd += u.costUsd;
|
|
37
46
|
}
|
|
38
|
-
|
|
47
|
+
}
|
|
48
|
+
return { ...(call === undefined ? state.total : state.calls.get(call) ?? { input: 0, output: 0, costUsd: 0 }) };
|
|
39
49
|
}
|
|
40
50
|
/** V8: Reached means either configured limit has been consumed, including truthful overshoot. */
|
|
41
51
|
export function reached(usage, budget) {
|
|
@@ -115,7 +115,7 @@ export class WorktreeIndex {
|
|
|
115
115
|
}
|
|
116
116
|
else if (e.type === "wf-created")
|
|
117
117
|
this.origins.set(journal, e.origin);
|
|
118
|
-
else if (isEntry(e, JT.attention) && e.item?.kind === "conflict") {
|
|
118
|
+
else if (isEntry(e, JT.attention) && e.item?.kind === "conflict" && String(e.item.id).startsWith("worktree:")) {
|
|
119
119
|
const held = this.reminders.get(e.item.id) ?? new Map();
|
|
120
120
|
this.reminders.set(e.item.id, held);
|
|
121
121
|
if (!held.has(journal))
|
|
@@ -6,10 +6,11 @@
|
|
|
6
6
|
// config{hash,config}: the orchestrator settings in effect from here; config-rejected{error,hash?}: a change of
|
|
7
7
|
// config.json refused, while the earlier settings stay (cleared by the next config record).
|
|
8
8
|
// orchestrator{version,pid} / orchestrator-exit{pid}: the orchestrator running (its package version) and its exit.
|
|
9
|
+
// writer-hold/writer-release{root,call}: the writer lock of a worktree root, held by one call until it ends.
|
|
9
10
|
import { foldExhaustion } from "./providers.js";
|
|
10
11
|
import { isEntry } from "../types.js";
|
|
11
12
|
export function emptyLedger() {
|
|
12
|
-
return { seen: 0, held: new Map(), observed: new Set(), skips: new Map(), exhausted: new Map() };
|
|
13
|
+
return { seen: 0, held: new Map(), writers: new Map(), observed: new Set(), skips: new Map(), exhausted: new Map() };
|
|
13
14
|
}
|
|
14
15
|
/** Apply one orchestrator ledger entry. */
|
|
15
16
|
export function applyLedger(state, e) {
|
|
@@ -31,7 +32,15 @@ export function applyLedger(state, e) {
|
|
|
31
32
|
else if (isEntry(e, "config-rejected"))
|
|
32
33
|
state.rejected = { error: String(e.error), ts: e.ts };
|
|
33
34
|
else if (isEntry(e, "orchestrator"))
|
|
34
|
-
state.orchestrator = { version: String(e.version), pid: Number(e.pid), ...(e.start ? { start: String(e.start) } : {}), ts: e.ts };
|
|
35
|
+
state.orchestrator = { version: String(e.version), pid: Number(e.pid), ...(e.start ? { start: String(e.start) } : {}), ts: e.ts, ...(e.restart === true ? { restart: true } : {}) };
|
|
36
|
+
else if (e.type === "writer-hold") {
|
|
37
|
+
if (!state.writers.has(String(e.root)))
|
|
38
|
+
state.writers.set(String(e.root), { call: String(e.call), ts: e.ts });
|
|
39
|
+
}
|
|
40
|
+
else if (e.type === "writer-release") {
|
|
41
|
+
if (state.writers.get(String(e.root))?.call === e.call)
|
|
42
|
+
state.writers.delete(String(e.root));
|
|
43
|
+
}
|
|
35
44
|
else if (isEntry(e, "orchestrator-exit")) {
|
|
36
45
|
if (state.orchestrator?.pid === e.pid)
|
|
37
46
|
state.orchestrator.exited = true;
|
|
@@ -6,24 +6,27 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
|
|
|
6
6
|
}
|
|
7
7
|
return path;
|
|
8
8
|
};
|
|
9
|
+
import { spawn } from 'node:child_process';
|
|
9
10
|
import { mkdir } from 'node:fs/promises';
|
|
10
11
|
import { fileURLToPath } from 'node:url';
|
|
11
12
|
import { realpathSync } from 'node:fs';
|
|
12
13
|
import { join, resolve } from 'node:path';
|
|
13
|
-
import { dsaHome, orchLedger, orchLock } from "../paths.js";
|
|
14
|
+
import { binDir, dsaHome, orchLedger, orchLock } from "../paths.js";
|
|
15
|
+
import { writeShim } from "../platform/lease.js";
|
|
14
16
|
import { openJournal } from "../kernel/journal.js";
|
|
15
17
|
import { OsLock } from "../platform/lock.js";
|
|
16
18
|
import { captureStart } from "../platform/proctable.js";
|
|
17
19
|
import { Engine } from "./engine.js";
|
|
18
20
|
import { configPath, configProblem, configStamp, recordConfig, stampedConfig, watchConfig } from "./config.js";
|
|
19
21
|
import { packageVersion } from "../version.js";
|
|
20
|
-
/** P2, C4, K6: Acquire the single authority before opening ledgers, recover, and idle-exit.
|
|
22
|
+
/** P2, C4, K6: Acquire the single authority before opening ledgers, recover, and idle-exit. Resolves `restart` when an
|
|
23
|
+
* applied restart request ended it (the caller starts the successor once the lock is released). */
|
|
21
24
|
export async function main(options = {}) {
|
|
22
25
|
const home = options.home ?? dsaHome();
|
|
23
26
|
await mkdir(home, { recursive: true });
|
|
24
27
|
const lock = await new OsLock().tryAcquire(orchLock(home));
|
|
25
28
|
if (!lock)
|
|
26
|
-
return;
|
|
29
|
+
return {};
|
|
27
30
|
let engine, ledgers, watcher;
|
|
28
31
|
try {
|
|
29
32
|
let config = options.config, stamp;
|
|
@@ -41,7 +44,14 @@ export async function main(options = {}) {
|
|
|
41
44
|
// Which version runs is visible to every pi session (status; a notice when it differs from the one pi loaded).
|
|
42
45
|
// `start` (Linux) tells this process from a later one given the same pid after a crash.
|
|
43
46
|
const start = await captureStart(process.pid).catch(() => '');
|
|
44
|
-
|
|
47
|
+
// Subagents reach `pi-durable-subagents hold …` through this shim, written before the start is recorded (executor puts its directory first on PATH).
|
|
48
|
+
try {
|
|
49
|
+
writeShim(binDir(home), process.execPath, fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? '../cli/main.ts' : '../cli/main.js', import.meta.url)));
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
console.error(`durable-subagents: cannot write ${binDir(home)}: ${String(error)}`);
|
|
53
|
+
}
|
|
54
|
+
await ledgers.orch.append('orchestrator', { version: packageVersion(), pid: process.pid, ...(start ? { start } : {}), restart: true });
|
|
45
55
|
const factory = options.executor ?? (await import(__rewriteRelativeImportExtension(new URL(import.meta.url.endsWith('.ts') ? './executor/index.ts' : './executor/index.js', import.meta.url).href))).default;
|
|
46
56
|
const executor = factory(ledgers);
|
|
47
57
|
engine = new Engine(ledgers, executor, options);
|
|
@@ -54,6 +64,7 @@ export async function main(options = {}) {
|
|
|
54
64
|
}
|
|
55
65
|
await engine.recover();
|
|
56
66
|
await engine.loop(options.signal);
|
|
67
|
+
return engine.restartRequested ? { restart: true } : {};
|
|
57
68
|
}
|
|
58
69
|
finally {
|
|
59
70
|
try {
|
|
@@ -77,5 +88,10 @@ if (process.argv[1] && realpathSync(resolve(process.argv[1])) === fileURLToPath(
|
|
|
77
88
|
const controller = new AbortController();
|
|
78
89
|
process.once('SIGTERM', () => controller.abort());
|
|
79
90
|
process.once('SIGINT', () => controller.abort());
|
|
80
|
-
main({ signal: controller.signal }).
|
|
91
|
+
main({ signal: controller.signal }).then(({ restart }) => {
|
|
92
|
+
// Restart: the successor runs this same entry, which an update replaced in place; any pi session or CLI would start
|
|
93
|
+
// it too (whoever takes the lock first wins, the others exit at once).
|
|
94
|
+
if (restart)
|
|
95
|
+
spawn(process.execPath, [process.argv[1]], { detached: true, stdio: 'ignore', env: process.env }).unref();
|
|
96
|
+
}).catch(error => { console.error(error); process.exitCode = 1; });
|
|
81
97
|
}
|
|
@@ -9,6 +9,7 @@ import { JT, isEntry } from "../types.js";
|
|
|
9
9
|
import { emptyLedger, foldLedger } from "./ledger.js";
|
|
10
10
|
import { packageVersion } from "../version.js";
|
|
11
11
|
import { worktreeCalls, worktreeLabel } from "./executor/worktree.js";
|
|
12
|
+
import { leaseCalls, leaseLines, leaseState } from "../platform/lease.js";
|
|
12
13
|
/** A workflow has live work: it runs, or follow-ups opened on it after it finished have not ended yet. */
|
|
13
14
|
export function isLive(wf) {
|
|
14
15
|
return wf.status === "running" || (wf.followUps ?? 0) > 0;
|
|
@@ -127,6 +128,13 @@ function snapshotReducer(wid, entries) {
|
|
|
127
128
|
delete call.hibernated;
|
|
128
129
|
hibernating.delete(call.callId);
|
|
129
130
|
}
|
|
131
|
+
else if (e.type === "writer-wait" || e.type === "writer-acquired") {
|
|
132
|
+
const call = calls.get(String(e.call)), holder = String(e.holder ?? "");
|
|
133
|
+
if (call && e.type === "writer-wait")
|
|
134
|
+
call.writerWait = { root: String(e.root), holder: holder.includes("/") ? `${holder.split("@")[0]}/${holder.split("/")[1].split("@")[0]}` : holder };
|
|
135
|
+
else if (call)
|
|
136
|
+
delete call.writerWait;
|
|
137
|
+
}
|
|
130
138
|
else if (e.type === "hibernated") {
|
|
131
139
|
// The decision to hibernate precedes the fence; the slot is released only once the execution is fenced.
|
|
132
140
|
const call = calls.get(String(e.call));
|
|
@@ -191,6 +199,7 @@ function snapshotReducer(wid, entries) {
|
|
|
191
199
|
call.result = e.result;
|
|
192
200
|
call.endedAt = e.ts;
|
|
193
201
|
delete call.hibernated;
|
|
202
|
+
delete call.writerWait;
|
|
194
203
|
hibernating.delete(call.callId);
|
|
195
204
|
}
|
|
196
205
|
else if (isEntry(e, JT.attention)) {
|
|
@@ -418,7 +427,7 @@ export function formatUsage(u) {
|
|
|
418
427
|
return `${n(u.input)} in / ${n(u.output)} out${u.costUsd > 0 ? `, $${u.costUsd.toFixed(u.costUsd < 0.01 ? 4 : 2)}` : ""}`;
|
|
419
428
|
}
|
|
420
429
|
/** P25, T10: Compact one workflow snapshot: per-call one-line facts, usage, open attention; no outputs or entries. */
|
|
421
|
-
export function compactWorkflow(wf) {
|
|
430
|
+
export function compactWorkflow(wf, leases) {
|
|
422
431
|
const progress = progressOf(wf); // v12 §4
|
|
423
432
|
return {
|
|
424
433
|
wid: wf.wid, ...(wf.name ? { name: wf.name } : {}), ...(wf.origin ? { origin: wf.origin } : {}), status: wf.status, rev: wf.rev,
|
|
@@ -430,7 +439,8 @@ export function compactWorkflow(wf) {
|
|
|
430
439
|
return { key: c.key, gen: c.gen, callId: c.callId, phase: c.phase, ...(r ? { status: r.status, ok: r.ok } : {}),
|
|
431
440
|
...(c.sharedWorktree ? { sharedWorktree: c.sharedWorktree } : {}),
|
|
432
441
|
...(c.model ? { model: c.model } : {}), ...(c.tools ? { tools: c.tools } : {}), ...(c.pending ? { pending: c.pending } : {}), ...(c.switching ? { switching: c.switching } : {}), ...(c.switchFailed ? { switchFailed: c.switchFailed } : {}), ...(nonzero(c.usage) ? { usage: c.usage } : {}),
|
|
433
|
-
...(last ? { lastLine: clip(last, 200) } : {}), ...(r?.error ? { error: clip(r.error, 300) } : {}), ...(c.hibernated ? { hibernated: true } : {})
|
|
442
|
+
...(last ? { lastLine: clip(last, 200) } : {}), ...(r?.error ? { error: clip(r.error, 300) } : {}), ...(c.hibernated ? { hibernated: true } : {}),
|
|
443
|
+
...(c.writerWait && !r ? { writerWait: c.writerWait } : {}), ...(!r && leases?.get(c.callId) ? { lease: leases.get(c.callId) } : {}) };
|
|
434
444
|
}),
|
|
435
445
|
attention: wf.attention.map(a => ({ id: a.id, rev: a.rev, kind: a.kind, text: clip(a.text, 300), ...(a.call ? { call: a.call } : {}), ...(a.qid ? { qid: a.qid } : {}) })),
|
|
436
446
|
...(wf.paused ? { paused: true } : {}), ...(wf.followUps ? { followUps: wf.followUps } : {}),
|
|
@@ -460,14 +470,14 @@ function snapshots(home, origin) {
|
|
|
460
470
|
/** P25, T10: Compact status of all workflows: own session first, then newest first; finished ones beyond the first `keep` collapse into a count. */
|
|
461
471
|
export function statusView(home, options = {}) {
|
|
462
472
|
const keep = options.keep ?? 10;
|
|
463
|
-
const { all, since } = snapshots(home, options.origin);
|
|
473
|
+
const { all, since } = snapshots(home, options.origin), leases = leaseState(home), byCall = leaseCalls(leases);
|
|
464
474
|
let finished = 0;
|
|
465
475
|
const shown = all.filter(w => !settled(w) || ++finished <= keep);
|
|
466
476
|
const hidden = all.length - shown.length;
|
|
467
477
|
const paused = all.filter(w => w.paused).length;
|
|
468
|
-
return { workflows: shown.map(compactWorkflow), ...(hidden ? { olderFinished: hidden, hint: "status wid=<wid> shows any workflow in detail" } : {}),
|
|
478
|
+
return { workflows: shown.map(w => compactWorkflow(w, byCall)), ...(hidden ? { olderFinished: hidden, hint: "status wid=<wid> shows any workflow in detail" } : {}),
|
|
469
479
|
...(paused ? { paused: `${paused} workflow${paused > 1 ? "s" : ""} paused (stop-all, drain or a quit pi) since ${new Date(since).toISOString()}; resume continues them (new runs are not affected)` } : {}),
|
|
470
|
-
...slotsView(home) };
|
|
480
|
+
...slotsView(home), ...(leases.length ? { leases: leaseLines(leases) } : {}) };
|
|
471
481
|
}
|
|
472
482
|
const FINAL = ["done", "failed", "stopped"];
|
|
473
483
|
/** Finished with nothing left running (a follow-up on a finished workflow is live work). */
|
|
@@ -484,13 +494,13 @@ const ledgerStates = new Map();
|
|
|
484
494
|
* its version is not the one this process loaded: running work stays on the version it started with. */
|
|
485
495
|
export function orchestratorView(state, loaded = packageVersion()) {
|
|
486
496
|
const o = state.orchestrator;
|
|
487
|
-
if (!o || o.exited || !
|
|
497
|
+
if (!o || o.exited || !processAlive(o.pid, o.start))
|
|
488
498
|
return {};
|
|
489
499
|
return { orchestrator: `${o.version} (pid ${o.pid})`, ...(o.version === loaded ? {} : { versionNote: versionNote(o.version, loaded) }) };
|
|
490
500
|
}
|
|
491
501
|
/** Whether the recorded orchestrator still runs. On Linux its start time also tells it from a later process given the
|
|
492
502
|
* same pid after a crash (elsewhere the pid alone is checked). */
|
|
493
|
-
function
|
|
503
|
+
export function processAlive(pid, start) {
|
|
494
504
|
try {
|
|
495
505
|
process.kill(pid, 0);
|
|
496
506
|
}
|
|
@@ -520,10 +530,9 @@ export function versionNote(running, loaded) {
|
|
|
520
530
|
return newer(running, loaded)
|
|
521
531
|
? `this pi session loaded durable-subagents ${loaded}, older than the running orchestrator ${running}; start a new pi session to use ${running}`
|
|
522
532
|
: `the orchestrator runs durable-subagents ${running}, this pi loaded ${loaded}: running work stays on ${running}. ` +
|
|
523
|
-
`It exits about 10 s after all work ends and starts again on
|
|
524
|
-
`
|
|
525
|
-
`
|
|
526
|
-
`A resume before it exits keeps ${running}, and pi sessions started before the update start ${running} again`;
|
|
533
|
+
`It exits about 10 s after all work ends and starts again on the installed version. To switch sooner: ` +
|
|
534
|
+
`restart (\`pi-durable-subagents restart\` or the subagents tool's restart action) — refused while an execution runs, ` +
|
|
535
|
+
`calls waiting for an answer or a slot do not block it; force fences running executions, which resume on the new version`;
|
|
527
536
|
}
|
|
528
537
|
/** orchestratorView of the home's orchestrator ledger. */
|
|
529
538
|
export function runningOrchestrator(home) {
|
|
@@ -552,7 +561,7 @@ export function slotsView(home, now = Date.now()) {
|
|
|
552
561
|
* The full view (every call's last line and usage) ran to tens of thousands of tokens on a busy home. */
|
|
553
562
|
export function statusBrief(home, options = {}) {
|
|
554
563
|
const keep = options.keep ?? 5, now = options.now ?? Date.now(), origin = options.origin;
|
|
555
|
-
const { all } = snapshots(home, origin);
|
|
564
|
+
const { all } = snapshots(home, origin), leases = leaseState(home), byCall = leaseCalls(leases, now);
|
|
556
565
|
const mine = (w) => !origin || w.origin === origin;
|
|
557
566
|
const line = (w) => {
|
|
558
567
|
const p = progressOf(w), notOk = latestCalls(w).filter(c => c.result && !c.result.ok).length;
|
|
@@ -569,7 +578,8 @@ export function statusBrief(home, options = {}) {
|
|
|
569
578
|
...(live && c.phase !== "asking" && quiet !== undefined && quiet >= 60_000 ? { quiet: age(quiet) } : {}),
|
|
570
579
|
...(live && c.startedAt !== undefined ? { tokens: tokens(c.usage) } : {}),
|
|
571
580
|
...(c.result ? { status: c.result.status, ...(c.result.error ? { error: clip(c.result.error, 200) } : {}) } : {}),
|
|
572
|
-
...(c.hibernated ? { hibernated: true } : {}),
|
|
581
|
+
...(c.hibernated ? { hibernated: true } : {}), ...(live && c.writerWait ? { writerWait: c.writerWait } : {}),
|
|
582
|
+
...(live && byCall.get(c.callId) ? { lease: byCall.get(c.callId) } : {}),
|
|
573
583
|
...(live && c.switching ? { switching: c.switching } : {}), ...(live && c.switchFailed ? { switchFailed: c.switchFailed } : {}) };
|
|
574
584
|
});
|
|
575
585
|
const asking = open.filter(a => a.kind === "question" && a.call).map(a => ({ to: `${w.wid}/${callKey(a.call)}`, ...(a.qid ? { qid: a.qid } : {}),
|
|
@@ -588,7 +598,7 @@ export function statusBrief(home, options = {}) {
|
|
|
588
598
|
return {
|
|
589
599
|
active: active.filter(mine).map(brief), ...(others.length ? { otherSessions: others.slice(0, 10).map(line) } : {}),
|
|
590
600
|
finished: finished.slice(0, keep).map(line), ...(finished.length > keep ? { olderFinished: finished.length - keep } : {}),
|
|
591
|
-
...(paused ? { paused } : {}), ...slotsView(home, now),
|
|
601
|
+
...(paused ? { paused } : {}), ...slotsView(home, now), ...(leases.length ? { leases: leaseLines(leases, now) } : {}),
|
|
592
602
|
hint: "status wid=<wid> shows one workflow (outputs clipped); add key=<key> for one call's full result, or full:true for everything",
|
|
593
603
|
};
|
|
594
604
|
}
|
|
@@ -105,14 +105,26 @@ export async function prepareRun(body, discovery, warnings = []) {
|
|
|
105
105
|
...(body.usageBudget ? { usageBudget: body.usageBudget } : {}), ...(body.maxCalls !== undefined ? { maxCalls: body.maxCalls } : {}) };
|
|
106
106
|
}
|
|
107
107
|
/** A1, A5: Reduce only the current revision, retaining all history for seals and allocation. */
|
|
108
|
+
const revisions = new WeakMap();
|
|
109
|
+
const terminals = new WeakMap();
|
|
108
110
|
export function revisionEntries(wf) {
|
|
109
|
-
const entries = wf.journal.entries()
|
|
110
|
-
|
|
111
|
+
const entries = wf.journal.entries();
|
|
112
|
+
let view = revisions.get(entries);
|
|
113
|
+
if (!view) {
|
|
114
|
+
const start = entries.findLastIndex(e => e.type === 'revised');
|
|
115
|
+
view = start < 0 ? entries : entries.slice(start);
|
|
116
|
+
revisions.set(entries, view);
|
|
117
|
+
}
|
|
118
|
+
return view;
|
|
111
119
|
}
|
|
112
120
|
/** A1, A5: A resume supersedes a terminal observation without deleting history. */
|
|
113
121
|
export function terminalEntry(entries) {
|
|
122
|
+
if (terminals.has(entries))
|
|
123
|
+
return terminals.get(entries);
|
|
114
124
|
const last = entries.findLast(e => e.type === JT.done || (e.type === 'resumed' && !e.call) || e.type === 'revised');
|
|
115
|
-
|
|
125
|
+
const terminal = last?.type === JT.done ? last : undefined;
|
|
126
|
+
terminals.set(entries, terminal);
|
|
127
|
+
return terminal;
|
|
116
128
|
}
|
|
117
129
|
/** Housekeeping: bytes under a path (files counted once by lstat; a missing path is 0). */
|
|
118
130
|
export async function diskUsage(path) {
|
package/dist/paths.js
CHANGED
|
@@ -6,6 +6,8 @@ import { ENV } from "./types.js";
|
|
|
6
6
|
export const dsaHome = (env = process.env) => env[ENV.home] || path.join(os.homedir(), ".pi", "durable-subagents");
|
|
7
7
|
export const orchLedger = (home) => path.join(home, "orchestrator.jsonl");
|
|
8
8
|
export const orchLock = (home) => path.join(home, "orchestrator.lock");
|
|
9
|
+
/** Executables for subagents: the orchestrator writes a `pi-durable-subagents` shim here and children get it on PATH. */
|
|
10
|
+
export const binDir = (home) => path.join(home, "bin");
|
|
9
11
|
/** Drop box for requests addressed to the orchestrator. */
|
|
10
12
|
export const orchInbox = (home) => path.join(home, "inbox");
|
|
11
13
|
/** Sender outboxes live under this root (kernel Outbox.open(root, ...)). */
|
|
@@ -3,12 +3,14 @@ import { PassThrough } from "node:stream";
|
|
|
3
3
|
import { setTimeout as delay } from "node:timers/promises";
|
|
4
4
|
import { performance } from "node:perf_hooks";
|
|
5
5
|
import { captureStart, ProcessTable } from "./proctable.js";
|
|
6
|
+
// One table per orchestrator process, including evaluator and gate observers.
|
|
7
|
+
const sharedTable = new ProcessTable();
|
|
6
8
|
const identity = (p) => `${p.pid}:${p.start}`;
|
|
7
9
|
/** P22, P23, C1, C3: Direct launch and identity-based retirement of executions. */
|
|
8
10
|
export class Containment {
|
|
9
11
|
table;
|
|
10
12
|
launched = new Map();
|
|
11
|
-
constructor(table =
|
|
13
|
+
constructor(table = sharedTable) { this.table = table; }
|
|
12
14
|
/** C1, C3: Inject the execution tag and capture the direct child's start token. */
|
|
13
15
|
async spawn(spec) {
|
|
14
16
|
const child = spawn(spec.command, spec.args, { cwd: spec.cwd,
|
|
@@ -44,16 +46,20 @@ export class Containment {
|
|
|
44
46
|
}
|
|
45
47
|
}
|
|
46
48
|
/** P22: Find tagged processes and the live descendants of known identities. */
|
|
47
|
-
async scan(known) {
|
|
49
|
+
async scan(known, options = {}) {
|
|
48
50
|
const execs = new Set([...known.keys(), ...this.launched.keys()]);
|
|
49
|
-
const all = await this.table.list(execs);
|
|
51
|
+
const all = await this.table.list(execs, options);
|
|
50
52
|
for (const p of all)
|
|
51
53
|
if (p.tag !== undefined)
|
|
52
54
|
execs.add(p.tag);
|
|
53
|
-
const result = new Map();
|
|
55
|
+
const result = new Map(), liveStarts = new Map(all.map(p => [p.pid, p.start]));
|
|
54
56
|
for (const exec of execs) {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
+
// Historical identities cannot select an absent/reused pid. Avoid allocating a string/Set entry for each one.
|
|
58
|
+
const ids = new Set();
|
|
59
|
+
for (const group of [known.get(exec) ?? [], this.launched.get(exec) ?? []])
|
|
60
|
+
for (const p of group)
|
|
61
|
+
if (p.start !== "" && liveStarts.get(p.pid) === p.start)
|
|
62
|
+
ids.add(identity(p));
|
|
57
63
|
const selected = new Map(all.filter(p => p.tag === exec || ids.has(identity(p))).map(p => [p.pid, p]));
|
|
58
64
|
let changed = true;
|
|
59
65
|
while (changed) {
|
|
@@ -79,7 +85,7 @@ export class Containment {
|
|
|
79
85
|
const remaining = deadline - performance.now();
|
|
80
86
|
let timer;
|
|
81
87
|
const live = await Promise.race([
|
|
82
|
-
this.scan(new Map([[exec, [...known.values()]]])),
|
|
88
|
+
this.scan(new Map([[exec, [...known.values()]]]), { fresh: true }),
|
|
83
89
|
new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`Fence timeout: ${exec}`)), Math.max(0, remaining)); }),
|
|
84
90
|
]).finally(() => clearTimeout(timer));
|
|
85
91
|
const targets = live.get(exec) ?? [];
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
// Command-scoped resource leases: `pi-durable-subagents hold <resource> -- <command>` (README "Resource leases").
|
|
2
|
+
// One ticket file per request under $DSA_HOME/leases/<resource>/, numbered under a kernel lock so the numbers are
|
|
3
|
+
// strictly increasing and never reused. Grants are strict FIFO by number: an exclusive request needs no earlier live
|
|
4
|
+
// ticket, a shared one no earlier live exclusive ticket. Whether a request is grantable depends only on earlier tickets,
|
|
5
|
+
// which can only disappear, so a grant needs no lock once the ticket exists. A ticket is live while its wrapper or its
|
|
6
|
+
// command (pid + start token) lives, or processes the command left in its group remain; a waiter next in line ends such
|
|
7
|
+
// leftovers of a killed wrapper. Anyone may delete a dead ticket. No orchestrator is involved.
|
|
8
|
+
import { chmodSync, mkdirSync, readdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
|
9
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { OsLock } from "./lock.js";
|
|
12
|
+
export const RESOURCE = /^[A-Za-z0-9._-]{1,64}$/;
|
|
13
|
+
export const leasesRoot = (home) => path.join(home, "leases");
|
|
14
|
+
export const leaseDir = (home, resource) => path.join(leasesRoot(home), resource);
|
|
15
|
+
const ticketName = (seq) => `${String(seq).padStart(12, "0")}.json`;
|
|
16
|
+
const TICKET = /^\d{12}\.json$/;
|
|
17
|
+
/** Whether a process still runs; on Linux the start token also tells it from a later process with the same pid. */
|
|
18
|
+
export function alive(p) {
|
|
19
|
+
if (!p)
|
|
20
|
+
return false;
|
|
21
|
+
try {
|
|
22
|
+
process.kill(p.pid, 0);
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
if (error.code !== "EPERM")
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
if (!p.start || process.platform !== "linux")
|
|
29
|
+
return true;
|
|
30
|
+
try {
|
|
31
|
+
const fields = readFileSync(`/proc/${p.pid}/stat`, "utf8");
|
|
32
|
+
const rest = fields.slice(fields.lastIndexOf(")") + 2).split(" ");
|
|
33
|
+
return rest[0] !== "Z" && rest[0] !== "X" && rest[19] === p.start;
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/** Whether process group `pgid` still has a member that can run (Linux: zombies, which no signal ends, do not count). */
|
|
40
|
+
export function groupAlive(pgid) {
|
|
41
|
+
try {
|
|
42
|
+
process.kill(-pgid, 0);
|
|
43
|
+
}
|
|
44
|
+
catch (e) {
|
|
45
|
+
if (e.code !== "EPERM")
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
if (process.platform !== "linux")
|
|
49
|
+
return true;
|
|
50
|
+
let pids;
|
|
51
|
+
try {
|
|
52
|
+
pids = readdirSync("/proc").filter(n => /^\d+$/.test(n));
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
57
|
+
for (const pid of pids) {
|
|
58
|
+
try {
|
|
59
|
+
const fields = readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
60
|
+
const rest = fields.slice(fields.lastIndexOf(")") + 2).split(" ");
|
|
61
|
+
if (Number(rest[2]) === pgid && rest[0] !== "Z" && rest[0] !== "X")
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
64
|
+
catch { /* exited meanwhile */ }
|
|
65
|
+
}
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Whether the command's process group outlives both the wrapper and the command (a killed wrapper cannot end what the
|
|
70
|
+
* command left behind). The command was spawned as its group's leader, so the group id is its pid. A pid stays
|
|
71
|
+
* allocated while a group uses it as its id, so with the leader gone a group with that id is the command's own. While a
|
|
72
|
+
* process owns the pid it must be the command itself (on Linux the start token, zombie or not); any other owner means
|
|
73
|
+
* the original group emptied and the pid was reused. macOS has no start token here: a reused pid that is alive there
|
|
74
|
+
* counts as the command (the lease waits for it), never as leftovers to end.
|
|
75
|
+
*/
|
|
76
|
+
export function orphaned(t) {
|
|
77
|
+
const c = t.command;
|
|
78
|
+
if (!c || alive(t.wrapper) || alive(c))
|
|
79
|
+
return false;
|
|
80
|
+
if (process.platform === "linux") {
|
|
81
|
+
try {
|
|
82
|
+
const fields = readFileSync(`/proc/${c.pid}/stat`, "utf8");
|
|
83
|
+
const rest = fields.slice(fields.lastIndexOf(")") + 2).split(" ");
|
|
84
|
+
if (!c.start || rest[19] !== c.start)
|
|
85
|
+
return false;
|
|
86
|
+
}
|
|
87
|
+
catch { /* the leader is gone: only its group can remain */ }
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
try {
|
|
91
|
+
process.kill(c.pid, 0);
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
catch { /* the leader is gone */ }
|
|
95
|
+
}
|
|
96
|
+
if (groupAlive(c.pid))
|
|
97
|
+
return true;
|
|
98
|
+
// One /proc listing is not atomic: a member can fork and exit between reading the directory and reading the stats, so
|
|
99
|
+
// "no running member" is not proof. Whatever the group still holds is leftovers of a holder that is gone (which the
|
|
100
|
+
// waiters would end anyway), so end it: a group signal reaches every member at once, a newborn included; zombies stay.
|
|
101
|
+
try {
|
|
102
|
+
process.kill(-c.pid, "SIGKILL");
|
|
103
|
+
}
|
|
104
|
+
catch { /* the group is gone */ }
|
|
105
|
+
return false;
|
|
106
|
+
}
|
|
107
|
+
/** A ticket is live while its wrapper, its command, or processes the command left in its group run. */
|
|
108
|
+
export const live = (t) => alive(t.wrapper) || alive(t.command) || orphaned(t);
|
|
109
|
+
/** The tickets of one resource in request order (unreadable or half-written files are skipped). */
|
|
110
|
+
export function readTickets(home, resource) {
|
|
111
|
+
let names;
|
|
112
|
+
try {
|
|
113
|
+
names = readdirSync(leaseDir(home, resource));
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
return [];
|
|
117
|
+
}
|
|
118
|
+
const out = [];
|
|
119
|
+
for (const name of names.filter(n => TICKET.test(n)).sort()) {
|
|
120
|
+
try {
|
|
121
|
+
out.push(JSON.parse(readFileSync(path.join(leaseDir(home, resource), name), "utf8")));
|
|
122
|
+
}
|
|
123
|
+
catch { /* raced with a delete */ }
|
|
124
|
+
}
|
|
125
|
+
return out;
|
|
126
|
+
}
|
|
127
|
+
/** Live tickets; dead ones are deleted (their numbers are never reused, so a stale reader cannot delete a newer one). */
|
|
128
|
+
export function liveTickets(home, resource) {
|
|
129
|
+
return readTickets(home, resource).filter(t => {
|
|
130
|
+
if (live(t))
|
|
131
|
+
return true;
|
|
132
|
+
try {
|
|
133
|
+
unlinkSync(path.join(leaseDir(home, resource), ticketName(t.seq)));
|
|
134
|
+
}
|
|
135
|
+
catch { /* already gone */ }
|
|
136
|
+
return false;
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
/** The live tickets that come before `t` and keep it waiting (empty when it may run). */
|
|
140
|
+
export function blockers(t, tickets) {
|
|
141
|
+
return tickets.filter(o => o.seq < t.seq && (t.mode === "exclusive" || o.mode === "exclusive"));
|
|
142
|
+
}
|
|
143
|
+
function writeTicketSync(home, t) {
|
|
144
|
+
const file = path.join(leaseDir(home, t.resource), ticketName(t.seq)), tmp = `${file}.${process.pid}.tmp`;
|
|
145
|
+
writeFileSync(tmp, JSON.stringify(t));
|
|
146
|
+
renameSync(tmp, file);
|
|
147
|
+
}
|
|
148
|
+
export async function writeTicket(home, t) {
|
|
149
|
+
const file = path.join(leaseDir(home, t.resource), ticketName(t.seq)), tmp = `${file}.${process.pid}.tmp`;
|
|
150
|
+
await writeFile(tmp, JSON.stringify(t));
|
|
151
|
+
renameSync(tmp, file);
|
|
152
|
+
}
|
|
153
|
+
export function removeTicket(home, t) {
|
|
154
|
+
try {
|
|
155
|
+
unlinkSync(path.join(leaseDir(home, t.resource), ticketName(t.seq)));
|
|
156
|
+
}
|
|
157
|
+
catch { /* already gone */ }
|
|
158
|
+
}
|
|
159
|
+
/** Queue a request: number it under the resource's kernel lock and write its ticket before the lock is released, so a
|
|
160
|
+
* later request always sees it. */
|
|
161
|
+
export async function enqueue(home, ticket, lock = new OsLock()) {
|
|
162
|
+
const dir = leaseDir(home, ticket.resource);
|
|
163
|
+
mkdirSync(dir, { recursive: true });
|
|
164
|
+
let handle = await lock.tryAcquire(path.join(dir, ".lock"));
|
|
165
|
+
for (let delay = 5; !handle; delay = Math.min(delay * 2, 100)) {
|
|
166
|
+
await new Promise(r => setTimeout(r, delay));
|
|
167
|
+
handle = await lock.tryAcquire(path.join(dir, ".lock"));
|
|
168
|
+
}
|
|
169
|
+
try {
|
|
170
|
+
const counter = path.join(dir, ".seq");
|
|
171
|
+
const last = Number.parseInt(await readFile(counter, "utf8").catch(() => "0"), 10) || 0;
|
|
172
|
+
const highest = readTickets(home, ticket.resource).reduce((m, t) => Math.max(m, t.seq), last);
|
|
173
|
+
const t = { ...ticket, seq: highest + 1 };
|
|
174
|
+
await writeFile(`${counter}.tmp`, String(t.seq));
|
|
175
|
+
renameSync(`${counter}.tmp`, counter);
|
|
176
|
+
writeTicketSync(home, t);
|
|
177
|
+
return t;
|
|
178
|
+
}
|
|
179
|
+
finally {
|
|
180
|
+
await handle.release();
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
/** Every resource with live tickets: holders (granted) and waiters, in request order. Cheap: a directory listing,
|
|
184
|
+
* small files and a liveness check of the few ticket processes. */
|
|
185
|
+
export function leaseState(home) {
|
|
186
|
+
let resources;
|
|
187
|
+
try {
|
|
188
|
+
resources = readdirSync(leasesRoot(home)).filter(r => RESOURCE.test(r)).sort();
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
return [];
|
|
192
|
+
}
|
|
193
|
+
return resources.map(resource => {
|
|
194
|
+
const tickets = liveTickets(home, resource);
|
|
195
|
+
return { resource, holders: tickets.filter(t => t.grantedAt !== undefined), waiters: tickets.filter(t => t.grantedAt === undefined) };
|
|
196
|
+
}).filter(r => r.holders.length || r.waiters.length);
|
|
197
|
+
}
|
|
198
|
+
const age = (ms) => ms < 60_000 ? `${Math.max(0, Math.round(ms / 1000))}s` : ms < 3_600_000 ? `${Math.round(ms / 60_000)}m` : `${(ms / 3_600_000).toFixed(1)}h`;
|
|
199
|
+
const clip = (s, n) => s.length > n ? `${s.slice(0, n - 1)}…` : s;
|
|
200
|
+
/** "<wid>/<key>" of a call id "<wid>@<rev>/<key>@<gen>". */
|
|
201
|
+
export const callAddress = (call) => { const m = /^([^@/]+)@\d+\/(.+)@\d+$/.exec(call); return m ? `${m[1]}/${m[2]}` : call; };
|
|
202
|
+
/** Who holds or waits: the subagent call when there is one, else the pid and command. */
|
|
203
|
+
export const who = (t) => t.call ? callAddress(t.call) : `pid ${t.wrapper.pid} \`${clip(t.argv.join(" "), 60)}\``;
|
|
204
|
+
/** One status line per resource: "machine held by <who> (exclusive, 12m, `make bench`); waiting: <who> 3m, …". */
|
|
205
|
+
export function leaseLines(state, now = Date.now()) {
|
|
206
|
+
return state.map(({ resource, holders, waiters }) => {
|
|
207
|
+
const held = holders.length
|
|
208
|
+
? `held by ${holders.map(t => `${who(t)} (${t.mode}, ${age(now - (t.grantedAt ?? t.since))}${t.call ? `, \`${clip(t.argv.join(" "), 60)}\`` : ""}${t.note ? `, ${clip(t.note, 80)}` : ""})`).join(", ")}`
|
|
209
|
+
: "free";
|
|
210
|
+
return `${resource} ${held}${waiters.length ? `; waiting: ${waiters.map(t => `${who(t)} ${t.mode === "shared" ? "shared " : ""}${age(now - t.since)}`).join(", ")}` : ""}`;
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
/** Per call id: "holds lease machine" / "waiting for lease machine 3m". */
|
|
214
|
+
export function leaseCalls(state, now = Date.now()) {
|
|
215
|
+
const out = new Map();
|
|
216
|
+
for (const { resource, holders, waiters } of state) {
|
|
217
|
+
for (const t of holders)
|
|
218
|
+
if (t.call)
|
|
219
|
+
out.set(t.call, [out.get(t.call), `holds lease ${resource}`].filter(Boolean).join(", "));
|
|
220
|
+
for (const t of waiters)
|
|
221
|
+
if (t.call)
|
|
222
|
+
out.set(t.call, [out.get(t.call), `waiting for lease ${resource} ${age(now - t.since)}`].filter(Boolean).join(", "));
|
|
223
|
+
}
|
|
224
|
+
return out;
|
|
225
|
+
}
|
|
226
|
+
const quote = (s) => `'${s.replace(/'/g, `'\\''`)}'`;
|
|
227
|
+
/** The `pi-durable-subagents` command for subagents: `<dir>/pi-durable-subagents` runs this node with the CLI entry of
|
|
228
|
+
* the running orchestrator, so children can `hold` without the package on their PATH. Rewritten at every start. */
|
|
229
|
+
export function writeShim(dir, node, entry) {
|
|
230
|
+
mkdirSync(dir, { recursive: true });
|
|
231
|
+
const file = path.join(dir, "pi-durable-subagents"), tmp = `${file}.${process.pid}.tmp`;
|
|
232
|
+
writeFileSync(tmp, `#!/bin/sh\nexec ${quote(node)} ${quote(entry)} "$@"\n`);
|
|
233
|
+
chmodSync(tmp, 0o755);
|
|
234
|
+
renameSync(tmp, file);
|
|
235
|
+
return file;
|
|
236
|
+
}
|