pi-durable-subagents 1.0.11 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +2 -2
- package/dist/agent/main/snapshots.js +2 -2
- package/dist/cli/doctor.js +3 -3
- package/dist/orchestrator/engine.js +4 -5
- package/dist/orchestrator/executor/effects/prepare.js +2 -2
- package/dist/orchestrator/executor/index.js +34 -37
- package/dist/orchestrator/executor/observe.js +26 -8
- package/dist/orchestrator/executor/sweep.js +3 -3
- package/dist/orchestrator/ledger.js +49 -0
- package/dist/orchestrator/main.js +2 -1
- package/dist/orchestrator/snapshot.js +12 -22
- package/dist/types.js +6 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.0.13
|
|
4
|
+
|
|
5
|
+
- A subagent resumed after its execution was interrupted (the orchestrator
|
|
6
|
+
restarted, the process died, a failover) is told that the processes its
|
|
7
|
+
tools had started, background ones included, were stopped with it, and not
|
|
8
|
+
to wait for them; an asker that hibernated while waiting is told the same
|
|
9
|
+
with the answer. Before, it was told only to continue, and one slept on a
|
|
10
|
+
test loop that no longer ran.
|
|
11
|
+
- The "no execution activity" alert names the tool command running and how
|
|
12
|
+
long it has run (for example "running bash `make fault-matrix` for 14m"), so
|
|
13
|
+
a long silent command reads differently from a stuck call.
|
|
14
|
+
|
|
15
|
+
## 1.0.12
|
|
16
|
+
|
|
17
|
+
- Slot, memory and model decisions read the orchestrator settings recorded in
|
|
18
|
+
its ledger, the same state `status` reports, through one fold of that ledger
|
|
19
|
+
shared by the executor and `status`; an orchestrator embedded with given
|
|
20
|
+
settings records them as well. Journal readers use typed entries instead of
|
|
21
|
+
casts. No change in behavior is intended.
|
|
22
|
+
|
|
3
23
|
## 1.0.11
|
|
4
24
|
|
|
5
25
|
- A gate whose processes outlive their fence is reported once: when the gate
|
package/README.md
CHANGED
|
@@ -51,7 +51,7 @@ the npx cache, so `install-service` refuses to run from there.
|
|
|
51
51
|
| A step runs past its `timeoutMs` (even inside a silent tool) | Stops it cleanly as `timeout`. Only time spent working counts; waiting for you does not. |
|
|
52
52
|
| You quit pi (Ctrl+D, `/quit`, closing the terminal) while subagents run | That session's workflows pause: nothing more is spent, nothing is lost. When you come back, pi says so; `resume` (or `r` in the list) continues them in the same sessions. Set `"onQuit": "continue"` to let them run on instead. |
|
|
53
53
|
| pi crashes or is killed (`kill -9`) while subagents run | The work keeps running. When you come back, the session that started the work is told what needs you. |
|
|
54
|
-
| The machine or the orchestrator dies mid-run | The next pi you open resumes the work. Finished results are kept and nothing runs twice. |
|
|
54
|
+
| The machine or the orchestrator dies mid-run | The next pi you open resumes the work. Finished results are kept and nothing runs twice. The resumed subagent is told that processes its tools had started (background ones included) were stopped, so it checks them instead of waiting for them. |
|
|
55
55
|
| You steer a subagent while it is asking you a question | Your message reaches it, in order. Nothing is rejected or lost. |
|
|
56
56
|
| Two steers arrive out of order and the second replaces the first | Only the second one applies. |
|
|
57
57
|
| A step is refused, or a dependency fails | The workflow stops that branch cleanly. Nothing is retried in vain. |
|
|
@@ -174,7 +174,7 @@ The main agent is interrupted only when there is something to decide:
|
|
|
174
174
|
|
|
175
175
|
- a question;
|
|
176
176
|
- a finished workflow;
|
|
177
|
-
- a stalled subagent;
|
|
177
|
+
- a stalled subagent (the alert names the command it is running and for how long, so a long silent command reads differently from a stuck call);
|
|
178
178
|
- an unknown outcome;
|
|
179
179
|
- a reached budget.
|
|
180
180
|
|
|
@@ -3,7 +3,7 @@ import { StringDecoder } from "node:string_decoder";
|
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { readJournalSnapshot } from "../../kernel/journal.js";
|
|
5
5
|
import { journalPath, orchLedger } from "../../paths.js";
|
|
6
|
-
import { CT, JT } from "../../types.js";
|
|
6
|
+
import { CT, JT, attentionEntries } from "../../types.js";
|
|
7
7
|
import { holdOf, ledgerIndex } from "../../orchestrator/snapshot.js";
|
|
8
8
|
/** P15, P25: Read workflow snapshots without modifying another domain's history; a wid with an orchestrator.jsonl
|
|
9
9
|
* `pruned{wid}` entry is gone (housekeeping), even while its directory is still being removed. */
|
|
@@ -76,7 +76,7 @@ function unresolved(entries) {
|
|
|
76
76
|
if (items)
|
|
77
77
|
return items;
|
|
78
78
|
const done = new Set(entries.filter(e => e.type === JT.attentionResolved).map(e => `${e.id}\0${e.rev}`));
|
|
79
|
-
items = entries
|
|
79
|
+
items = attentionEntries(entries).map(e => e.item).filter(item => !done.has(`${item.id}\0${item.rev}`));
|
|
80
80
|
open.set(entries, items);
|
|
81
81
|
return items;
|
|
82
82
|
}
|
package/dist/cli/doctor.js
CHANGED
|
@@ -11,7 +11,7 @@ import { OsLock } from "../platform/lock.js";
|
|
|
11
11
|
import { journalPath, orchInbox, orchLedger, orchLock } from "../paths.js";
|
|
12
12
|
import { snapshotFromEntries } from "../orchestrator/snapshot.js";
|
|
13
13
|
import { diskUsage } from "../orchestrator/store.js";
|
|
14
|
-
import { JT } from "../types.js";
|
|
14
|
+
import { JT, isEntry } from "../types.js";
|
|
15
15
|
import { serviceFiles } from "./service.js";
|
|
16
16
|
const HOUR = 3_600_000, DAY = 24 * HOUR, CLI = "pi-durable-subagents";
|
|
17
17
|
const quote = (s) => `'${s.replace(/'/g, `'\\''`)}'`;
|
|
@@ -63,8 +63,8 @@ export async function doctor(home, env = process.env, now = Date.now()) {
|
|
|
63
63
|
// Finished notices are delivered to the origin session and stay open in the journal by design: not counted.
|
|
64
64
|
const resolved = new Set(entries.filter(e => e.type === JT.attentionResolved).map(e => `${e.id}@${e.rev}`));
|
|
65
65
|
for (const e of entries) {
|
|
66
|
-
const item = e.item;
|
|
67
|
-
if (
|
|
66
|
+
const item = isEntry(e, JT.attention) ? e.item : undefined;
|
|
67
|
+
if (item && item.kind !== "finished" && !resolved.has(`${item.id}@${item.rev}`) && now - e.ts > HOUR)
|
|
68
68
|
report.attention.push({ wid, id: item.id, kind: item.kind, ageMs: now - e.ts });
|
|
69
69
|
if (e.type !== "fence-failed")
|
|
70
70
|
continue;
|
|
@@ -17,7 +17,7 @@ import { contentHash } from "../kernel/ids.js";
|
|
|
17
17
|
import { planDecisions, reduceLifecycle } from "../kernel/lifecycle.js";
|
|
18
18
|
import { scanInbox } from "../kernel/mailbox.js";
|
|
19
19
|
import { orchInbox, pinnedDir } from "../paths.js";
|
|
20
|
-
import { JT } from "../types.js";
|
|
20
|
+
import { JT, attentionEntries, isEntry } from "../types.js";
|
|
21
21
|
import { EvaluatorClient } from "./evaluator-client.js";
|
|
22
22
|
import { Store, revisionEntries, terminalEntry } from "./store.js";
|
|
23
23
|
import { formatUsage, holdOf, refusedResult, snapshotFromEntries } from "./snapshot.js";
|
|
@@ -509,7 +509,7 @@ export class Engine {
|
|
|
509
509
|
void this.executor.run(ticket).then(() => this.background(async () => {
|
|
510
510
|
if (!wf.journal.entries().some(e => e.type === JT.sealed && e.call === id))
|
|
511
511
|
throw new Error(`Generation returned without seal: ${id}`);
|
|
512
|
-
if (!wf.journal.entries().some(e => e
|
|
512
|
+
if (!wf.journal.entries().some(e => isEntry(e, JT.attention) && e.item.id === `finished:${id}`))
|
|
513
513
|
await wf.journal.append(JT.attention, { item: { id: `finished:${id}`, rev: 1, kind: 'finished', wid: wf.wid, call: id, text: finishedText(wf.wid, wf.journal.entries(), id), origin: wf.origin } });
|
|
514
514
|
this.generations.delete(id);
|
|
515
515
|
}), error => {
|
|
@@ -686,13 +686,12 @@ export class Engine {
|
|
|
686
686
|
async attention(wf) {
|
|
687
687
|
const done = this.terminal(wf);
|
|
688
688
|
const rev = wf.journal.entries().filter(e => e.type === JT.done).length;
|
|
689
|
-
if (done && !wf.journal.entries().some(e => e
|
|
689
|
+
if (done && !wf.journal.entries().some(e => isEntry(e, JT.attention) && e.item.id === `finished:${wf.wid}` && e.item.rev === rev)) {
|
|
690
690
|
await wf.journal.append(JT.attention, { item: { id: `finished:${wf.wid}`, rev, kind: 'finished', wid: wf.wid, text: finishedText(wf.wid, wf.journal.entries()) } });
|
|
691
691
|
}
|
|
692
692
|
}
|
|
693
693
|
async resolveFinished(wf) {
|
|
694
|
-
for (const
|
|
695
|
-
const item = e.item;
|
|
694
|
+
for (const { item } of attentionEntries(wf.journal.entries())) {
|
|
696
695
|
if (item.kind === 'finished' && !wf.journal.entries().some(r => r.type === JT.attentionResolved && r.id === item.id && r.rev === item.rev)) {
|
|
697
696
|
await wf.journal.append(JT.attentionResolved, { id: item.id, rev: item.rev, resolution: 'resumed' });
|
|
698
697
|
}
|
|
@@ -8,14 +8,14 @@ import { basename, dirname, join, resolve, sep } from "node:path";
|
|
|
8
8
|
import { randomUUID } from "node:crypto";
|
|
9
9
|
import { publishFile } from "../../../kernel/mailbox.js";
|
|
10
10
|
import { contentHash } from "../../../kernel/ids.js";
|
|
11
|
-
import { JT } from "../../../types.js";
|
|
11
|
+
import { JT, isEntry } from "../../../types.js";
|
|
12
12
|
const exec = promisify(execFile);
|
|
13
13
|
const git = async (cwd, ...args) => (await exec("git", args, { cwd, timeout: 30000, maxBuffer: 4 * 1024 * 1024 })).stdout.trim();
|
|
14
14
|
const records = (t, type) => t.journal.entries().filter(e => e.type === type && e.call === t.callId);
|
|
15
15
|
/** P15, A5: Publish one stable attention item for a retained or conflicting effect. */
|
|
16
16
|
export async function attention(t, kind, text) {
|
|
17
17
|
const id = `${kind}:${t.callId}`;
|
|
18
|
-
if (!t.journal.entries().some(e => e
|
|
18
|
+
if (!t.journal.entries().some(e => isEntry(e, JT.attention) && e.item?.id === id))
|
|
19
19
|
await t.journal.append(JT.attention, { item: { id, rev: 1, kind: "unknown", text, wid: t.wid, call: t.callId } });
|
|
20
20
|
}
|
|
21
21
|
/** Git lists worktrees by real path (macOS: /var is /private/var); compare paths in that form. */
|
|
@@ -11,7 +11,7 @@ import { mkdir, open, readdir, readFile, rm, unlink, writeFile } from "node:fs/p
|
|
|
11
11
|
import { homedir } from "node:os";
|
|
12
12
|
import { join } from "node:path";
|
|
13
13
|
import { fileURLToPath } from "node:url";
|
|
14
|
-
import { CT, JT } from "../../types.js";
|
|
14
|
+
import { CT, JT, attentionEntries, isEntry } from "../../types.js";
|
|
15
15
|
import { callDir, callInbox, callSession, outboxRoot } from "../../paths.js";
|
|
16
16
|
import { Outbox } from "../../kernel/mailbox.js";
|
|
17
17
|
import { contentHash, forwardRid } from "../../kernel/ids.js";
|
|
@@ -22,7 +22,7 @@ import { buildCallResult } from "../../compat/result.js";
|
|
|
22
22
|
import createEffects from "./effects/index.js";
|
|
23
23
|
import { continueSession } from "./generation.js";
|
|
24
24
|
import { hibernation, openQuestion } from "./hibernate.js";
|
|
25
|
-
import {
|
|
25
|
+
import { emptyLedger, foldLedger, holdings as holdingsOf, settingsOf } from "../ledger.js";
|
|
26
26
|
import { evidence, fatalProviderError, quotaExhausted, refusedByProvider, forgetSession, readSessionState, receiptId, sessionModel } from "./session.js";
|
|
27
27
|
import { activeTotal } from "./time.js";
|
|
28
28
|
import { observeExecution } from "./observe.js";
|
|
@@ -88,6 +88,16 @@ async function defaultModel() {
|
|
|
88
88
|
return settings.defaultModel ? `${settings.defaultProvider ? `${settings.defaultProvider}/` : ""}${settings.defaultModel}` : undefined;
|
|
89
89
|
}
|
|
90
90
|
/** P2, P9, P22: Construct the journal-owned execution authority under the engine's OS lock. */
|
|
91
|
+
/** Restart feedback: a resumed child believed the test loop it had started was still running and slept on it. Its
|
|
92
|
+
* previous execution was stopped (fenced), and with it every process its tools started, background ones included. */
|
|
93
|
+
export function continueMessage(dangling) {
|
|
94
|
+
return "Your previous execution was interrupted and this one continues the task. Processes your tools started " +
|
|
95
|
+
"(background ones included) were stopped with it: do not wait for them or their output; check what they left " +
|
|
96
|
+
"and start again what is still needed. Tool calls whose outcomes are unknown: " + (dangling.join(", ") || "none") + ".";
|
|
97
|
+
}
|
|
98
|
+
/** An asker hibernates while it waits (its execution is stopped to free the provider slot), so the answer says so. */
|
|
99
|
+
export const HIBERNATED_NOTE = "While you waited for the answer below your execution was stopped; processes your tools had " +
|
|
100
|
+
"started (background ones included) were stopped with it. Check them before relying on them.";
|
|
91
101
|
export default function createExecutor(ledgers, options = {}) {
|
|
92
102
|
const { home, config, orch } = ledgers;
|
|
93
103
|
const containment = options.containment ?? serialContainment(), effects = options.effects ?? createEffects(ledgers);
|
|
@@ -115,25 +125,12 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
115
125
|
};
|
|
116
126
|
const wake = () => { for (const fn of waiters)
|
|
117
127
|
fn(); waiters.clear(); };
|
|
118
|
-
// F3: fold only orchestrator entries appended since the last fold (
|
|
119
|
-
const ledger =
|
|
120
|
-
const folded = () =>
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
if (e.type === "hold")
|
|
125
|
-
ledger.held.set(id, e);
|
|
126
|
-
else if (e.type === "release" && ledger.held.get(id)?.exec === e.exec)
|
|
127
|
-
ledger.held.delete(id);
|
|
128
|
-
else if (e.type === "switch-observed")
|
|
129
|
-
ledger.observed.add(`${e.exec}\n${e.rid}`);
|
|
130
|
-
else if (e.type === "skip")
|
|
131
|
-
ledger.skips.set(`${e.pool}\n${e.model}`, Math.max(Number(e.until), ledger.skips.get(`${e.pool}\n${e.model}`) ?? 0));
|
|
132
|
-
foldExhaustion(ledger.exhausted, e);
|
|
133
|
-
}
|
|
134
|
-
return ledger;
|
|
135
|
-
};
|
|
136
|
-
const holdings = () => [...folded().held.values()];
|
|
128
|
+
// F3, A1: fold only orchestrator entries appended since the last fold (ledger.ts, shared with `status`).
|
|
129
|
+
const ledger = emptyLedger();
|
|
130
|
+
const folded = () => foldLedger(ledger, orch.entries());
|
|
131
|
+
/** A4: guards read the settings recorded in the ledger (config.json changes are recorded between admissions). */
|
|
132
|
+
const settings = () => settingsOf(folded(), config);
|
|
133
|
+
const holdings = () => holdingsOf(folded());
|
|
137
134
|
const skipped = (pool, model) => (folded().skips.get(`${pool}\n${model.provider}/${model.id}`) ?? 0) > Date.now();
|
|
138
135
|
/** A provider whose usage window is used up admits no call until its next try, and then one probe at a time. */
|
|
139
136
|
const unavailable = (provider) => {
|
|
@@ -349,13 +346,12 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
349
346
|
}
|
|
350
347
|
async function workflowReached(t) {
|
|
351
348
|
const hit = reached(totalUsage(t.journal.entries()), t.workflowBudget), id = `budget:${t.wid}`;
|
|
352
|
-
if (hit && !t.journal.entries().some(e => e
|
|
349
|
+
if (hit && !t.journal.entries().some(e => isEntry(e, JT.attention) && e.item.id === id))
|
|
353
350
|
await t.journal.append(JT.attention, { item: { id, rev: 1, kind: "budget", text: "Workflow budget reached", wid: t.wid } });
|
|
354
351
|
return hit;
|
|
355
352
|
}
|
|
356
353
|
async function retireAttention(journal, call) {
|
|
357
|
-
for (const
|
|
358
|
-
const item = e.item;
|
|
354
|
+
for (const { item } of attentionEntries(journal.entries())) {
|
|
359
355
|
if (item.kind === "finished" || item.id === `unknown:${call}` || item.id.startsWith("fence:"))
|
|
360
356
|
continue;
|
|
361
357
|
if (item.call === call && !journal.entries().some(r => r.type === JT.attentionResolved && r.id === item.id && r.rev === item.rev))
|
|
@@ -365,7 +361,7 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
365
361
|
/** P15, AC4: A call sealed `unknown` raises exactly one unknown item for its origin; seal and recovery both run this. */
|
|
366
362
|
async function unknownAttention(journal, call) {
|
|
367
363
|
const result = sealed(journal, call), id = `unknown:${call}`;
|
|
368
|
-
if (result?.status !== "unknown" || journal.entries().some(e => e
|
|
364
|
+
if (result?.status !== "unknown" || journal.entries().some(e => isEntry(e, JT.attention) && e.item.id === id))
|
|
369
365
|
return;
|
|
370
366
|
const text = `Call ${call} ended with an unknown outcome: ${result.error || "no evidence of what it did"}. It was not re-run; check its effects before continuing.`;
|
|
371
367
|
await journal.append(JT.attention, { item: { id, rev: 1, kind: "unknown", text, wid: address(call).wid, call } });
|
|
@@ -454,15 +450,15 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
454
450
|
continue;
|
|
455
451
|
const probe = provider !== undefined && folded().exhausted.has(provider);
|
|
456
452
|
const holders = holdings().filter(e => e.pool === provider);
|
|
457
|
-
const limit =
|
|
453
|
+
const limit = settings().providers?.[provider ?? ""]?.slots ?? Infinity;
|
|
458
454
|
if (!capacity({ kind: "provider", holders: holders.length, capacity: limit }))
|
|
459
455
|
continue;
|
|
460
456
|
// A burst of dispatches all read the same MemAvailable before any child has grown: subtract the children
|
|
461
457
|
// admitted in the last 30 s (their memory is not visible yet), so a burst cannot over-commit the headroom.
|
|
462
|
-
const perChild =
|
|
458
|
+
const perChild = settings().memory?.perChildMb ?? 300;
|
|
463
459
|
const warming = holdings().filter(e => e.pool === "memory" && Date.now() - e.ts < MEM_WARMUP_MS).length;
|
|
464
460
|
const measured = await (options.memory ?? availableMemory)(), available = measured - warming * perChild;
|
|
465
|
-
const admitted = capacity({ kind: "memory", available, reserve:
|
|
461
|
+
const admitted = capacity({ kind: "memory", available, reserve: settings().memory?.reserveMb ?? 2048, perChild });
|
|
466
462
|
// F3: every admitted dispatch is recorded; repeated refusals at most once per 30 s per call.
|
|
467
463
|
if (admitted || a.refusedAt === undefined || Date.now() - a.refusedAt >= MEM_RECORD_MS) {
|
|
468
464
|
await orch.append("mem", { available, admitted, exec, ...(warming ? { measured, warming } : {}) });
|
|
@@ -508,9 +504,10 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
508
504
|
const recorded = sessionModel(entries);
|
|
509
505
|
const ownSegment = entries.some(e => e.type === "custom" && e.customType === CT.exec && typeof e.data?.exec === "string" && e.data.exec.startsWith(`${t.callId}#`));
|
|
510
506
|
const freshFork = t.spec.context === "fork" && !t.continueFrom && !ownSegment;
|
|
511
|
-
const
|
|
512
|
-
const
|
|
513
|
-
const
|
|
507
|
+
const { defaultModel: configured, pools } = settings();
|
|
508
|
+
const raw = t.spec.model ?? t.agent.model ?? configured ?? await defaultModel();
|
|
509
|
+
const pool = raw && pools?.[raw] ? raw : undefined;
|
|
510
|
+
const candidates = raw ? resolveModel(raw, pools) : [{ id: "" }];
|
|
514
511
|
const candidate = recorded && candidates.some(m => m.provider === recorded.provider && m.id === recorded.id);
|
|
515
512
|
// Leave the session's model for the pool's others when its pool skips it after losses, or its provider's usage
|
|
516
513
|
// window is used up; and at a new generation, go back to the pool's order of preference.
|
|
@@ -537,7 +534,7 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
537
534
|
if (typeof qid !== "string" || typeof rev !== "number")
|
|
538
535
|
continue;
|
|
539
536
|
const id = `q:${t.callId}:${qid}`;
|
|
540
|
-
if (!t.journal.entries().some(r => r
|
|
537
|
+
if (!t.journal.entries().some(r => isEntry(r, JT.attention) && r.item.id === id && r.item.rev === rev))
|
|
541
538
|
await t.journal.append(JT.attention, { item: { id, rev, kind: "question", text: String(question), wid: t.wid, call: t.callId, qid, session: callSession(home, t.wid, t.key, t.gen) } });
|
|
542
539
|
const answered = entries.some(r => {
|
|
543
540
|
const details = r.message?.details ?? r.details;
|
|
@@ -755,7 +752,7 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
755
752
|
else if (unresolved && !receipt)
|
|
756
753
|
await sender.send(t.callId, "continue", { message: String(unresolved.message) }, { qid: String(unresolved.qid), rev: Number(unresolved.rev) }, { rid: String(unresolved.rid2) });
|
|
757
754
|
else if (previous)
|
|
758
|
-
await sender.send(t.callId, "continue", { message:
|
|
755
|
+
await sender.send(t.callId, "continue", { message: continueMessage(dangling) }, undefined, { rid: contentHash([exec, "dispatch"]) });
|
|
759
756
|
else
|
|
760
757
|
await sender.send(t.callId, "task", { message: t.opening?.message ?? t.spec.task }, undefined, { rid: contentHash([exec, "dispatch"]) });
|
|
761
758
|
if (interrupted(a)) {
|
|
@@ -783,7 +780,7 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
783
780
|
if (!q || interrupted(a) || segment < 0 || !entries.slice(segment + 1).some(e => e.customType === CT.question && e.data?.qid === q.qid) || journal.entries().some(e => e.type === "answer-bound" && e.call === t.callId && e.qid === q.qid && e.rev === q.rev))
|
|
784
781
|
return;
|
|
785
782
|
await serial(async () => {
|
|
786
|
-
const attention = journal.entries().find(e => e
|
|
783
|
+
const attention = journal.entries().find(e => isEntry(e, JT.attention) && e.item.qid === q.qid && e.item.call === t.callId && e.item.rev === q.rev);
|
|
787
784
|
if (attention && Date.now() - attention.ts >= (config.k?.hibernateMs ?? 120000) && !has(journal, JT.fenced, exec) && hibernation(journal, t.callId)?.exec !== exec) {
|
|
788
785
|
await journal.append("hibernated", { call: t.callId, exec, qid: q.qid, rev: q.rev });
|
|
789
786
|
a.wake();
|
|
@@ -870,8 +867,8 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
870
867
|
if (sleeping.qid !== req.cond?.qid || sleeping.rev !== req.cond?.rev)
|
|
871
868
|
return { action: "reject", reason: "stale-rev" };
|
|
872
869
|
const rid2 = forwardRid(req.rid, ctx.widRev, ctx.key, hash);
|
|
873
|
-
const item = ctx.journal.entries().find(e => e.
|
|
874
|
-
await ctx.journal.append("answer-bound", { call: dest, qid: sleeping.qid, rev: sleeping.rev, rid: req.rid, rid2, hash, message:
|
|
870
|
+
const item = attentionEntries(ctx.journal.entries()).find(e => e.item.call === dest && e.item.qid === sleeping.qid && e.item.rev === sleeping.rev)?.item;
|
|
871
|
+
await ctx.journal.append("answer-bound", { call: dest, qid: sleeping.qid, rev: sleeping.rev, rid: req.rid, rid2, hash, message: `${HIBERNATED_NOTE}\n\nQuestion: ${item?.text ?? sleeping.qid}\nAnswer: ${req.body.message ?? ""}` });
|
|
875
872
|
active.get(dest)?.wake();
|
|
876
873
|
wake();
|
|
877
874
|
return { action: "apply" };
|
|
@@ -904,7 +901,7 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
904
901
|
const held = holdings().filter(h => h.exec === exec);
|
|
905
902
|
if (!held.some(h => h.pool === provider)) {
|
|
906
903
|
const target = holdings().filter(h => h.pool === provider);
|
|
907
|
-
if (!capacity({ kind: "provider", holders: target.length, capacity:
|
|
904
|
+
if (!capacity({ kind: "provider", holders: target.length, capacity: settings().providers?.[provider]?.slots ?? Infinity }))
|
|
908
905
|
return { action: "reject", reason: "provider-full" };
|
|
909
906
|
let slot = 0;
|
|
910
907
|
while (target.some(h => h.slot === slot))
|
|
@@ -5,18 +5,27 @@ import { join } from "node:path";
|
|
|
5
5
|
import { stat as fileStat } from "node:fs/promises";
|
|
6
6
|
import { performance } from "node:perf_hooks";
|
|
7
7
|
import { createInterface } from "node:readline";
|
|
8
|
-
import { JT } from "../../types.js";
|
|
8
|
+
import { JT, attentionEntries } from "../../types.js";
|
|
9
9
|
import { callDir, callSession } from "../../paths.js";
|
|
10
10
|
import { monotoneTime } from "../../kernel/guards.js";
|
|
11
11
|
import { readSession } from "./session.js";
|
|
12
12
|
import { ActiveTime, activeTotal } from "./time.js";
|
|
13
13
|
import { observation, reached, sessionUsage, totalUsage } from "./usage.js";
|
|
14
|
+
/** A tool call's command for a status line: bash's command, else its arguments, on one line and clipped. */
|
|
15
|
+
export function toolCommand(args) {
|
|
16
|
+
const a = args && typeof args === "object" ? args : {};
|
|
17
|
+
const text = typeof a.command === "string" ? a.command : Object.keys(a).length ? JSON.stringify(a) : "";
|
|
18
|
+
const line = text.replace(/\s+/g, " ").trim();
|
|
19
|
+
return line.length > 120 ? `${line.slice(0, 119)}…` : line;
|
|
20
|
+
}
|
|
14
21
|
/** P18, P31, P9: Observe slim evidence, decide limits and fence before returning to settlement. */
|
|
15
22
|
export async function observeExecution(d) {
|
|
16
23
|
const { home, config, ticket: t, exec, child, serial } = d;
|
|
17
24
|
const session = callSession(home, t.wid, t.key, t.gen), clock = new ActiveTime(), started = clock.last;
|
|
18
25
|
let progress = started, providerError;
|
|
19
26
|
const tools = new Set();
|
|
27
|
+
// The tool calls running now, for the stall text: a silent long command and a stuck call read differently.
|
|
28
|
+
const running = new Map();
|
|
20
29
|
const prior = activeTotal(t.journal.entries(), t.callId);
|
|
21
30
|
let size = (await fileStat(session).catch(() => ({ size: 0 }))).size, checkpoint = performance.now();
|
|
22
31
|
let signal;
|
|
@@ -42,19 +51,24 @@ export async function observeExecution(d) {
|
|
|
42
51
|
};
|
|
43
52
|
const stall = () => serial(async () => {
|
|
44
53
|
const id = `stall:${t.callId}`;
|
|
45
|
-
const items = t.journal.entries()
|
|
46
|
-
const last = items.at(-1)?.item;
|
|
54
|
+
const items = attentionEntries(t.journal.entries(), id), last = items.at(-1)?.item;
|
|
47
55
|
const open = last && !t.journal.entries().some(e => e.type === JT.attentionResolved && e.id === id && e.rev === last.rev);
|
|
48
56
|
const fresh = items.at(-1)?.exec === exec ? clock.last > Number(items.at(-1)?.horizon) : clock.last > started;
|
|
49
57
|
if (open && fresh)
|
|
50
58
|
await t.journal.append(JT.attentionResolved, { id, rev: last.rev, resolution: "activity" });
|
|
51
59
|
else if (!open && !clock.asking && performance.now() - clock.last >= (config.k?.stallMs ?? 600000))
|
|
52
|
-
await t.journal.append(JT.attention, { exec, horizon: clock.last, item: { id, rev: (last?.rev ?? 0) + 1, kind: "stall", text: `${t.wid}/${t.key}: no execution activity for ${Math.floor((performance.now() - clock.last) / 60000)}m
|
|
60
|
+
await t.journal.append(JT.attention, { exec, horizon: clock.last, item: { id, rev: (last?.rev ?? 0) + 1, kind: "stall", text: `${t.wid}/${t.key}: no execution activity for ${Math.floor((performance.now() - clock.last) / 60000)}m` + runningText(), wid: t.wid, call: t.callId } });
|
|
53
61
|
});
|
|
62
|
+
/** "; running bash `make matrix` for 14m (no output or CPU use seen)": a silent long command, not a stuck model. */
|
|
63
|
+
const runningText = () => {
|
|
64
|
+
const now = performance.now(), open = [...running.values()].filter(r => r.name !== "ask");
|
|
65
|
+
if (!open.length)
|
|
66
|
+
return "";
|
|
67
|
+
return "; running " + open.map(r => `${r.name}${r.command ? ` \`${r.command}\`` : ""} for ${Math.floor((now - r.since) / 60000)}m`).join(", ") + " (no output or CPU use seen)";
|
|
68
|
+
};
|
|
54
69
|
const noProgress = () => serial(async () => {
|
|
55
70
|
const id = `noprogress:${t.callId}`;
|
|
56
|
-
const items = t.journal.entries()
|
|
57
|
-
const last = items.at(-1)?.item;
|
|
71
|
+
const items = attentionEntries(t.journal.entries(), id), last = items.at(-1)?.item;
|
|
58
72
|
const open = last && !t.journal.entries().some(e => e.type === JT.attentionResolved && e.id === id && e.rev === last.rev);
|
|
59
73
|
const fresh = items.at(-1)?.exec === exec ? progress > Number(items.at(-1)?.horizon) : progress > started;
|
|
60
74
|
if (open && fresh)
|
|
@@ -96,10 +110,14 @@ export async function observeExecution(d) {
|
|
|
96
110
|
event.type === "message_end" && message?.stopReason === "error" ? message.errorMessage : undefined;
|
|
97
111
|
if (typeof error === "string" && error)
|
|
98
112
|
providerError = error;
|
|
99
|
-
if (event.type === "tool_execution_start")
|
|
113
|
+
if (event.type === "tool_execution_start") {
|
|
100
114
|
tools.add(String(event.toolCallId ?? ""));
|
|
101
|
-
|
|
115
|
+
running.set(String(event.toolCallId ?? ""), { name: String(event.toolName ?? "tool"), command: toolCommand(event.args), since: performance.now() });
|
|
116
|
+
}
|
|
117
|
+
if (event.type === "tool_execution_end") {
|
|
102
118
|
tools.delete(String(event.toolCallId ?? ""));
|
|
119
|
+
running.delete(String(event.toolCallId ?? ""));
|
|
120
|
+
}
|
|
103
121
|
// Receipt-time progress is independent of RPC chatter, CPU and session growth; open tools suppress alerts.
|
|
104
122
|
if (event.type === "message_update" || ["tool_execution_start", "tool_execution_update", "tool_execution_end"].includes(String(event.type)) ||
|
|
105
123
|
event.type === "message_end" && message?.stopReason !== "error" && (message?.usage?.output ?? 0) > 0)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { JT } from "../../types.js";
|
|
1
|
+
import { JT, attentionEntries, isEntry } from "../../types.js";
|
|
2
2
|
import { Containment as PlatformContainment } from "../../platform/containment.js";
|
|
3
3
|
import { ProcessTable } from "../../platform/proctable.js";
|
|
4
4
|
/** C2, F4: A containment whose process-table snapshots never overlap (ProcessTable.list is not reentrant). */
|
|
@@ -41,7 +41,7 @@ async function fenceFailure(journal, id, call, error) {
|
|
|
41
41
|
if (!journal.entries().some(e => e.type === "fence-failed" && e.exec === id))
|
|
42
42
|
await journal.append("fence-failed", { exec: id, error: String(error) });
|
|
43
43
|
const item = `fence:${id}`;
|
|
44
|
-
if (journal.entries().some(e => e
|
|
44
|
+
if (journal.entries().some(e => isEntry(e, JT.attention) && e.item.id === item))
|
|
45
45
|
return;
|
|
46
46
|
const text = id.startsWith("gate:")
|
|
47
47
|
? `Processes of gate ${id} did not exit after SIGKILL (${String(error)}). Its outcome stays unknown and is recorded once they are gone; check for stuck processes (e.g. blocked I/O).`
|
|
@@ -54,7 +54,7 @@ export function resolveFenceAttention(journal, id) {
|
|
|
54
54
|
}
|
|
55
55
|
/** The same, inside a `recordOnce` section. */
|
|
56
56
|
export async function fenceAttentionResolved(journal, id) {
|
|
57
|
-
const item = journal.entries()
|
|
57
|
+
const item = attentionEntries(journal.entries(), `fence:${id}`)[0]?.item;
|
|
58
58
|
if (item && !journal.entries().some(e => e.type === JT.attentionResolved && e.id === `fence:${id}` && e.rev === item.rev))
|
|
59
59
|
await journal.append(JT.attentionResolved, { id: `fence:${id}`, rev: item.rev, resolution: "fenced" });
|
|
60
60
|
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// A1, A4: the state of the orchestrator ledger that admission reads and `status` shows, folded by one reducer.
|
|
2
|
+
// hold/release{pool,slot,exec}: provider and memory slots held (a release frees only its holder's slot).
|
|
3
|
+
// switch-observed{exec,rid}: a model switch the execution's provider requests confirmed.
|
|
4
|
+
// skip{pool,model,until}: a pool candidate skipped after repeated losses (K7).
|
|
5
|
+
// provider-*: used-up usage windows (providers.ts).
|
|
6
|
+
// config{hash,config}: the orchestrator settings in effect from here; config-rejected{error,hash?}: a change of
|
|
7
|
+
// config.json refused, while the earlier settings stay (cleared by the next config record).
|
|
8
|
+
import { foldExhaustion } from "./providers.js";
|
|
9
|
+
import { isEntry } from "../types.js";
|
|
10
|
+
export function emptyLedger() {
|
|
11
|
+
return { seen: 0, held: new Map(), observed: new Set(), skips: new Map(), exhausted: new Map() };
|
|
12
|
+
}
|
|
13
|
+
/** Apply one orchestrator ledger entry. */
|
|
14
|
+
export function applyLedger(state, e) {
|
|
15
|
+
if (isEntry(e, "hold"))
|
|
16
|
+
state.held.set(`${e.pool}:${e.slot}`, e);
|
|
17
|
+
else if (isEntry(e, "release")) {
|
|
18
|
+
const id = `${e.pool}:${e.slot}`;
|
|
19
|
+
if (state.held.get(id)?.exec === e.exec)
|
|
20
|
+
state.held.delete(id);
|
|
21
|
+
}
|
|
22
|
+
else if (isEntry(e, "switch-observed"))
|
|
23
|
+
state.observed.add(`${e.exec}\n${e.rid}`);
|
|
24
|
+
else if (isEntry(e, "skip"))
|
|
25
|
+
state.skips.set(`${e.pool}\n${e.model}`, Math.max(Number(e.until), state.skips.get(`${e.pool}\n${e.model}`) ?? 0));
|
|
26
|
+
else if (isEntry(e, "config")) {
|
|
27
|
+
state.config = { hash: String(e.hash), settings: e.config, ts: e.ts };
|
|
28
|
+
delete state.rejected;
|
|
29
|
+
}
|
|
30
|
+
else if (isEntry(e, "config-rejected"))
|
|
31
|
+
state.rejected = { error: String(e.error), ts: e.ts };
|
|
32
|
+
foldExhaustion(state.exhausted, e);
|
|
33
|
+
}
|
|
34
|
+
/** Fold the entries appended since `state` was last folded (the ledger only grows); a ledger that is not the one
|
|
35
|
+
* folded so far (shorter, or another entry where the last folded one was) is folded from the start. */
|
|
36
|
+
export function foldLedger(state, entries) {
|
|
37
|
+
// Journal readers keep the entry objects of a ledger as it grows (kernel/journal.ts), so identity tells them apart.
|
|
38
|
+
if (state.seen && entries[state.seen - 1] !== state.last)
|
|
39
|
+
Object.assign(state, emptyLedger(), { config: undefined, rejected: undefined, last: undefined });
|
|
40
|
+
for (; state.seen < entries.length; state.seen++)
|
|
41
|
+
applyLedger(state, entries[state.seen]);
|
|
42
|
+
state.last = entries[state.seen - 1];
|
|
43
|
+
return state;
|
|
44
|
+
}
|
|
45
|
+
/** The provider slots held now (memory slots included). */
|
|
46
|
+
export const holdings = (state) => [...state.held.values()];
|
|
47
|
+
/** A4: the settings guards read: the ones recorded last, else (a ledger that records none: an embedded or test
|
|
48
|
+
* orchestrator given its settings) the given ones. */
|
|
49
|
+
export const settingsOf = (state, given) => state.config?.settings ?? given;
|
|
@@ -39,8 +39,9 @@ export async function main(options = {}) {
|
|
|
39
39
|
const factory = options.executor ?? (await import(__rewriteRelativeImportExtension(new URL(import.meta.url.endsWith('.ts') ? './executor/index.ts' : './executor/index.js', import.meta.url).href))).default;
|
|
40
40
|
const executor = factory(ledgers);
|
|
41
41
|
engine = new Engine(ledgers, executor, options);
|
|
42
|
+
// A4: admission reads the settings recorded last, so the ones this run starts with are recorded (given ones too).
|
|
43
|
+
await recordConfig(ledgers.orch, config);
|
|
42
44
|
if (stamp !== undefined) {
|
|
43
|
-
await recordConfig(ledgers.orch, config);
|
|
44
45
|
// A change applies between slot admissions, so one admission never mixes two versions of the limits.
|
|
45
46
|
watcher = watchConfig({ path: configPath(home), stamp, config, orch: ledgers.orch, intervalMs: Math.min(1000, config.k?.trackerMs ?? 1000),
|
|
46
47
|
apply: change => executor.reconfigure ? executor.reconfigure(change) : change() });
|
|
@@ -5,8 +5,8 @@ import { join } from "node:path";
|
|
|
5
5
|
import { compileFanout } from "../compat/fanout.js";
|
|
6
6
|
import { readJournalSnapshot } from "../kernel/journal.js";
|
|
7
7
|
import { journalPath, orchLedger, pinnedDir, workflowDir } from "../paths.js";
|
|
8
|
-
import { JT } from "../types.js";
|
|
9
|
-
import {
|
|
8
|
+
import { JT, isEntry } from "../types.js";
|
|
9
|
+
import { emptyLedger, foldLedger } from "./ledger.js";
|
|
10
10
|
/** A workflow has live work: it runs, or follow-ups opened on it after it finished have not ended yet. */
|
|
11
11
|
export function isLive(wf) {
|
|
12
12
|
return wf.status === "running" || (wf.followUps ?? 0) > 0;
|
|
@@ -191,7 +191,7 @@ function snapshotReducer(wid, entries) {
|
|
|
191
191
|
delete call.hibernated;
|
|
192
192
|
hibernating.delete(call.callId);
|
|
193
193
|
}
|
|
194
|
-
else if (e
|
|
194
|
+
else if (isEntry(e, JT.attention)) {
|
|
195
195
|
const item = e.item;
|
|
196
196
|
attention.push(item);
|
|
197
197
|
}
|
|
@@ -469,34 +469,24 @@ const tokens = (u) => { const n = u ? u.input + u.output : 0; return n >= 1e6 ?
|
|
|
469
469
|
const latestCalls = (wf) => [...new Map(wf.calls.map(c => [c.key, c])).values()];
|
|
470
470
|
/** Provider slots from the orchestrator ledger: holders per provider (hold/release{pool,slot,exec}) and the limits of
|
|
471
471
|
* the settings in effect (the latest config{hash,config}); config-rejected after it is reported too. */
|
|
472
|
+
// One fold per orchestrator ledger, extended as it grows (ledger.ts; the executor folds the same entries the same way).
|
|
473
|
+
const ledgerStates = new Map();
|
|
472
474
|
export function slotsView(home, now = Date.now()) {
|
|
473
|
-
const
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
if (e.type === "hold")
|
|
477
|
-
held.set(`${e.pool}:${e.slot}`, e);
|
|
478
|
-
else if (e.type === "release" && held.get(`${e.pool}:${e.slot}`)?.exec === e.exec)
|
|
479
|
-
held.delete(`${e.pool}:${e.slot}`);
|
|
480
|
-
else if (e.type === "config") {
|
|
481
|
-
config = e;
|
|
482
|
-
rejected = undefined;
|
|
483
|
-
}
|
|
484
|
-
else if (e.type === "config-rejected")
|
|
485
|
-
rejected = e;
|
|
486
|
-
foldExhaustion(used, e);
|
|
487
|
-
}
|
|
475
|
+
const path = orchLedger(home), state = foldLedger(ledgerStates.get(path) ?? emptyLedger(), readJournalSnapshot(path));
|
|
476
|
+
ledgerStates.set(path, state);
|
|
477
|
+
const { held, config, rejected } = state, used = state.exhausted;
|
|
488
478
|
const exhausted = [...used].sort(([a], [b]) => a.localeCompare(b)).map(([p, x]) => `${p} exhausted since ${age(now - x.since)} ago (${clip(x.error, 80)}), ` +
|
|
489
479
|
(x.probe ? `probing with ${x.probe.split("#")[0]}` : x.nextTry > now ? `next try in ${age(x.nextTry - now)}` : "next call probes it"));
|
|
490
|
-
const limits =
|
|
480
|
+
const limits = config?.settings.providers ?? {};
|
|
491
481
|
const holders = new Map();
|
|
492
482
|
for (const e of held.values())
|
|
493
483
|
if (e.pool !== "memory")
|
|
494
|
-
holders.set(
|
|
484
|
+
holders.set(e.pool, (holders.get(e.pool) ?? 0) + 1);
|
|
495
485
|
const names = [...new Set([...Object.keys(limits), ...holders.keys()])].sort();
|
|
496
486
|
const slots = names.map(p => { const n = holders.get(p) ?? 0, limit = limits[p]?.slots; return typeof limit === "number" ? `${p} ${n}/${limit}` : `${p} ${n} (no limit)`; });
|
|
497
|
-
return { ...(slots.length ? { slots } : {}), ...(config ? { config: `${
|
|
487
|
+
return { ...(slots.length ? { slots } : {}), ...(config ? { config: `${config.hash} since ${age(now - config.ts)} ago` } : {}),
|
|
498
488
|
...(exhausted.length ? { exhausted } : {}),
|
|
499
|
-
...(rejected ? { configRejected: `${clip(
|
|
489
|
+
...(rejected ? { configRejected: `${clip(rejected.error, 200)} (${age(now - rejected.ts)} ago); ${config ? config.hash : "the start settings"} stay in effect` } : {}) };
|
|
500
490
|
}
|
|
501
491
|
/** Tool status without a wid: what runs, what waits for an answer and what failed, with finished workflows one line each.
|
|
502
492
|
* The full view (every call's last line and usage) ran to tens of thousands of tokens on a busy home. */
|
package/dist/types.js
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
// Pinned shared contracts for pi-durable-subagents.
|
|
2
2
|
// Changing anything here needs parent approval: every module and test depends on it.
|
|
3
3
|
// Design references (repo-local, not published): design/principles.md (A1-A5, V1-V8), design/spec.md (P1-P38, K1-K11).
|
|
4
|
+
/** Narrow an entry to its type's fields (their shape is the writer's contract, not checked at runtime). */
|
|
5
|
+
export function isEntry(e, type) { return e.type === type; }
|
|
6
|
+
/** P15: the attention entries of a journal, of one item id when given (each revision of an item is an entry). */
|
|
7
|
+
export function attentionEntries(entries, id) {
|
|
8
|
+
return entries.filter((e) => isEntry(e, "attention") && (id === undefined || e.item.id === id));
|
|
9
|
+
}
|
|
4
10
|
// ---------------------------------------------------------------------------
|
|
5
11
|
// Child session entries (pi session file is the child's log; P4, P8, P23, P24)
|
|
6
12
|
// customType values written by the session agent. All carry details/data as below.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-durable-subagents",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.13",
|
|
4
4
|
"description": "Subagents for pi that never lose work and never do it twice. Crash-safe workflows, automatic recovery, and a live view just like the main agent.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|