pi-durable-subagents 1.0.10 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/agent/main/snapshots.js +2 -2
- package/dist/cli/doctor.js +3 -3
- package/dist/orchestrator/engine.js +4 -5
- package/dist/orchestrator/executor/effects/gate.js +9 -6
- package/dist/orchestrator/executor/effects/prepare.js +2 -2
- package/dist/orchestrator/executor/index.js +22 -35
- package/dist/orchestrator/executor/observe.js +3 -5
- package/dist/orchestrator/executor/sweep.js +24 -5
- package/dist/orchestrator/ledger.js +49 -0
- package/dist/orchestrator/main.js +2 -1
- package/dist/orchestrator/snapshot.js +12 -22
- package/dist/types.js +6 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.0.12
|
|
4
|
+
|
|
5
|
+
- Slot, memory and model decisions read the orchestrator settings recorded in
|
|
6
|
+
its ledger, the same state `status` reports, through one fold of that ledger
|
|
7
|
+
shared by the executor and `status`; an orchestrator embedded with given
|
|
8
|
+
settings records them as well. Journal readers use typed entries instead of
|
|
9
|
+
casts. No change in behavior is intended.
|
|
10
|
+
|
|
11
|
+
## 1.0.11
|
|
12
|
+
|
|
13
|
+
- A gate whose processes outlive their fence is reported once: when the gate
|
|
14
|
+
and the background sweep both recorded the failure at the same moment, a
|
|
15
|
+
workflow could show the same "processes may still run" attention item twice
|
|
16
|
+
(and the failure, the gate's unknown outcome and the resolution likewise).
|
|
17
|
+
|
|
3
18
|
## 1.0.10
|
|
4
19
|
|
|
5
20
|
- Provider failover for a used-up usage window. An error such as `503 No
|
|
@@ -3,7 +3,7 @@ import { StringDecoder } from "node:string_decoder";
|
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { readJournalSnapshot } from "../../kernel/journal.js";
|
|
5
5
|
import { journalPath, orchLedger } from "../../paths.js";
|
|
6
|
-
import { CT, JT } from "../../types.js";
|
|
6
|
+
import { CT, JT, attentionEntries } from "../../types.js";
|
|
7
7
|
import { holdOf, ledgerIndex } from "../../orchestrator/snapshot.js";
|
|
8
8
|
/** P15, P25: Read workflow snapshots without modifying another domain's history; a wid with an orchestrator.jsonl
|
|
9
9
|
* `pruned{wid}` entry is gone (housekeeping), even while its directory is still being removed. */
|
|
@@ -76,7 +76,7 @@ function unresolved(entries) {
|
|
|
76
76
|
if (items)
|
|
77
77
|
return items;
|
|
78
78
|
const done = new Set(entries.filter(e => e.type === JT.attentionResolved).map(e => `${e.id}\0${e.rev}`));
|
|
79
|
-
items = entries
|
|
79
|
+
items = attentionEntries(entries).map(e => e.item).filter(item => !done.has(`${item.id}\0${item.rev}`));
|
|
80
80
|
open.set(entries, items);
|
|
81
81
|
return items;
|
|
82
82
|
}
|
package/dist/cli/doctor.js
CHANGED
|
@@ -11,7 +11,7 @@ import { OsLock } from "../platform/lock.js";
|
|
|
11
11
|
import { journalPath, orchInbox, orchLedger, orchLock } from "../paths.js";
|
|
12
12
|
import { snapshotFromEntries } from "../orchestrator/snapshot.js";
|
|
13
13
|
import { diskUsage } from "../orchestrator/store.js";
|
|
14
|
-
import { JT } from "../types.js";
|
|
14
|
+
import { JT, isEntry } from "../types.js";
|
|
15
15
|
import { serviceFiles } from "./service.js";
|
|
16
16
|
const HOUR = 3_600_000, DAY = 24 * HOUR, CLI = "pi-durable-subagents";
|
|
17
17
|
const quote = (s) => `'${s.replace(/'/g, `'\\''`)}'`;
|
|
@@ -63,8 +63,8 @@ export async function doctor(home, env = process.env, now = Date.now()) {
|
|
|
63
63
|
// Finished notices are delivered to the origin session and stay open in the journal by design: not counted.
|
|
64
64
|
const resolved = new Set(entries.filter(e => e.type === JT.attentionResolved).map(e => `${e.id}@${e.rev}`));
|
|
65
65
|
for (const e of entries) {
|
|
66
|
-
const item = e.item;
|
|
67
|
-
if (
|
|
66
|
+
const item = isEntry(e, JT.attention) ? e.item : undefined;
|
|
67
|
+
if (item && item.kind !== "finished" && !resolved.has(`${item.id}@${item.rev}`) && now - e.ts > HOUR)
|
|
68
68
|
report.attention.push({ wid, id: item.id, kind: item.kind, ageMs: now - e.ts });
|
|
69
69
|
if (e.type !== "fence-failed")
|
|
70
70
|
continue;
|
|
@@ -17,7 +17,7 @@ import { contentHash } from "../kernel/ids.js";
|
|
|
17
17
|
import { planDecisions, reduceLifecycle } from "../kernel/lifecycle.js";
|
|
18
18
|
import { scanInbox } from "../kernel/mailbox.js";
|
|
19
19
|
import { orchInbox, pinnedDir } from "../paths.js";
|
|
20
|
-
import { JT } from "../types.js";
|
|
20
|
+
import { JT, attentionEntries, isEntry } from "../types.js";
|
|
21
21
|
import { EvaluatorClient } from "./evaluator-client.js";
|
|
22
22
|
import { Store, revisionEntries, terminalEntry } from "./store.js";
|
|
23
23
|
import { formatUsage, holdOf, refusedResult, snapshotFromEntries } from "./snapshot.js";
|
|
@@ -509,7 +509,7 @@ export class Engine {
|
|
|
509
509
|
void this.executor.run(ticket).then(() => this.background(async () => {
|
|
510
510
|
if (!wf.journal.entries().some(e => e.type === JT.sealed && e.call === id))
|
|
511
511
|
throw new Error(`Generation returned without seal: ${id}`);
|
|
512
|
-
if (!wf.journal.entries().some(e => e
|
|
512
|
+
if (!wf.journal.entries().some(e => isEntry(e, JT.attention) && e.item.id === `finished:${id}`))
|
|
513
513
|
await wf.journal.append(JT.attention, { item: { id: `finished:${id}`, rev: 1, kind: 'finished', wid: wf.wid, call: id, text: finishedText(wf.wid, wf.journal.entries(), id), origin: wf.origin } });
|
|
514
514
|
this.generations.delete(id);
|
|
515
515
|
}), error => {
|
|
@@ -686,13 +686,12 @@ export class Engine {
|
|
|
686
686
|
async attention(wf) {
|
|
687
687
|
const done = this.terminal(wf);
|
|
688
688
|
const rev = wf.journal.entries().filter(e => e.type === JT.done).length;
|
|
689
|
-
if (done && !wf.journal.entries().some(e => e
|
|
689
|
+
if (done && !wf.journal.entries().some(e => isEntry(e, JT.attention) && e.item.id === `finished:${wf.wid}` && e.item.rev === rev)) {
|
|
690
690
|
await wf.journal.append(JT.attention, { item: { id: `finished:${wf.wid}`, rev, kind: 'finished', wid: wf.wid, text: finishedText(wf.wid, wf.journal.entries()) } });
|
|
691
691
|
}
|
|
692
692
|
}
|
|
693
693
|
async resolveFinished(wf) {
|
|
694
|
-
for (const
|
|
695
|
-
const item = e.item;
|
|
694
|
+
for (const { item } of attentionEntries(wf.journal.entries())) {
|
|
696
695
|
if (item.kind === 'finished' && !wf.journal.entries().some(r => r.type === JT.attentionResolved && r.id === item.id && r.rev === item.rev)) {
|
|
697
696
|
await wf.journal.append(JT.attentionResolved, { id: item.id, rev: item.rev, resolution: 'resumed' });
|
|
698
697
|
}
|
|
@@ -5,15 +5,17 @@ import { join } from "node:path";
|
|
|
5
5
|
import { publishFile } from "../../../kernel/mailbox.js";
|
|
6
6
|
import { callDir } from "../../../paths.js";
|
|
7
7
|
import { validate } from "../../../agent/child/schema.js";
|
|
8
|
-
import { recordFenceFailure,
|
|
8
|
+
import { fenceAttentionResolved, recordFenceFailure, recordOnce } from "../sweep.js";
|
|
9
9
|
const tracked = (journal, id) => journal.entries().filter(e => e.type === "gate-tracked" && e.id === id).map(e => e.process);
|
|
10
10
|
// Calls parked on an unfenced gate, woken when the sweep records the gate's outcome.
|
|
11
11
|
const parked = new Map();
|
|
12
12
|
/** P30, F1: A gate proven retired after a failed fence gets its unknown outcome once and its attention resolved. */
|
|
13
13
|
export async function gateRetired(journal, id) {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
await recordOnce(journal, async () => {
|
|
15
|
+
if (!journal.entries().some(e => e.type === "gate" && e.id === id))
|
|
16
|
+
await journal.append("gate", { id, unknown: true });
|
|
17
|
+
await fenceAttentionResolved(journal, id);
|
|
18
|
+
});
|
|
17
19
|
for (const wake of parked.get(id) ?? [])
|
|
18
20
|
wake();
|
|
19
21
|
}
|
|
@@ -52,8 +54,9 @@ async function fenceGate(journal, intent, containment) {
|
|
|
52
54
|
console.error(`durable-subagents: fence of ${id} failed: ${String(error)}`);
|
|
53
55
|
if (/Fence timeout/.test(String(error)))
|
|
54
56
|
await recordFenceFailure(journal, id, String(intent.call), error);
|
|
55
|
-
else
|
|
56
|
-
await journal.
|
|
57
|
+
else
|
|
58
|
+
await recordOnce(journal, async () => { if (!journal.entries().some(e => e.type === "fence-failed" && e.exec === id))
|
|
59
|
+
await journal.append("fence-failed", { exec: id, error: String(error) }); });
|
|
57
60
|
return false;
|
|
58
61
|
}
|
|
59
62
|
}
|
|
@@ -8,14 +8,14 @@ import { basename, dirname, join, resolve, sep } from "node:path";
|
|
|
8
8
|
import { randomUUID } from "node:crypto";
|
|
9
9
|
import { publishFile } from "../../../kernel/mailbox.js";
|
|
10
10
|
import { contentHash } from "../../../kernel/ids.js";
|
|
11
|
-
import { JT } from "../../../types.js";
|
|
11
|
+
import { JT, isEntry } from "../../../types.js";
|
|
12
12
|
const exec = promisify(execFile);
|
|
13
13
|
const git = async (cwd, ...args) => (await exec("git", args, { cwd, timeout: 30000, maxBuffer: 4 * 1024 * 1024 })).stdout.trim();
|
|
14
14
|
const records = (t, type) => t.journal.entries().filter(e => e.type === type && e.call === t.callId);
|
|
15
15
|
/** P15, A5: Publish one stable attention item for a retained or conflicting effect. */
|
|
16
16
|
export async function attention(t, kind, text) {
|
|
17
17
|
const id = `${kind}:${t.callId}`;
|
|
18
|
-
if (!t.journal.entries().some(e => e
|
|
18
|
+
if (!t.journal.entries().some(e => isEntry(e, JT.attention) && e.item?.id === id))
|
|
19
19
|
await t.journal.append(JT.attention, { item: { id, rev: 1, kind: "unknown", text, wid: t.wid, call: t.callId } });
|
|
20
20
|
}
|
|
21
21
|
/** Git lists worktrees by real path (macOS: /var is /private/var); compare paths in that form. */
|
|
@@ -11,7 +11,7 @@ import { mkdir, open, readdir, readFile, rm, unlink, writeFile } from "node:fs/p
|
|
|
11
11
|
import { homedir } from "node:os";
|
|
12
12
|
import { join } from "node:path";
|
|
13
13
|
import { fileURLToPath } from "node:url";
|
|
14
|
-
import { CT, JT } from "../../types.js";
|
|
14
|
+
import { CT, JT, attentionEntries, isEntry } from "../../types.js";
|
|
15
15
|
import { callDir, callInbox, callSession, outboxRoot } from "../../paths.js";
|
|
16
16
|
import { Outbox } from "../../kernel/mailbox.js";
|
|
17
17
|
import { contentHash, forwardRid } from "../../kernel/ids.js";
|
|
@@ -22,7 +22,7 @@ import { buildCallResult } from "../../compat/result.js";
|
|
|
22
22
|
import createEffects from "./effects/index.js";
|
|
23
23
|
import { continueSession } from "./generation.js";
|
|
24
24
|
import { hibernation, openQuestion } from "./hibernate.js";
|
|
25
|
-
import {
|
|
25
|
+
import { emptyLedger, foldLedger, holdings as holdingsOf, settingsOf } from "../ledger.js";
|
|
26
26
|
import { evidence, fatalProviderError, quotaExhausted, refusedByProvider, forgetSession, readSessionState, receiptId, sessionModel } from "./session.js";
|
|
27
27
|
import { activeTotal } from "./time.js";
|
|
28
28
|
import { observeExecution } from "./observe.js";
|
|
@@ -115,25 +115,12 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
115
115
|
};
|
|
116
116
|
const wake = () => { for (const fn of waiters)
|
|
117
117
|
fn(); waiters.clear(); };
|
|
118
|
-
// F3: fold only orchestrator entries appended since the last fold (
|
|
119
|
-
const ledger =
|
|
120
|
-
const folded = () =>
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
if (e.type === "hold")
|
|
125
|
-
ledger.held.set(id, e);
|
|
126
|
-
else if (e.type === "release" && ledger.held.get(id)?.exec === e.exec)
|
|
127
|
-
ledger.held.delete(id);
|
|
128
|
-
else if (e.type === "switch-observed")
|
|
129
|
-
ledger.observed.add(`${e.exec}\n${e.rid}`);
|
|
130
|
-
else if (e.type === "skip")
|
|
131
|
-
ledger.skips.set(`${e.pool}\n${e.model}`, Math.max(Number(e.until), ledger.skips.get(`${e.pool}\n${e.model}`) ?? 0));
|
|
132
|
-
foldExhaustion(ledger.exhausted, e);
|
|
133
|
-
}
|
|
134
|
-
return ledger;
|
|
135
|
-
};
|
|
136
|
-
const holdings = () => [...folded().held.values()];
|
|
118
|
+
// F3, A1: fold only orchestrator entries appended since the last fold (ledger.ts, shared with `status`).
|
|
119
|
+
const ledger = emptyLedger();
|
|
120
|
+
const folded = () => foldLedger(ledger, orch.entries());
|
|
121
|
+
/** A4: guards read the settings recorded in the ledger (config.json changes are recorded between admissions). */
|
|
122
|
+
const settings = () => settingsOf(folded(), config);
|
|
123
|
+
const holdings = () => holdingsOf(folded());
|
|
137
124
|
const skipped = (pool, model) => (folded().skips.get(`${pool}\n${model.provider}/${model.id}`) ?? 0) > Date.now();
|
|
138
125
|
/** A provider whose usage window is used up admits no call until its next try, and then one probe at a time. */
|
|
139
126
|
const unavailable = (provider) => {
|
|
@@ -349,13 +336,12 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
349
336
|
}
|
|
350
337
|
async function workflowReached(t) {
|
|
351
338
|
const hit = reached(totalUsage(t.journal.entries()), t.workflowBudget), id = `budget:${t.wid}`;
|
|
352
|
-
if (hit && !t.journal.entries().some(e => e
|
|
339
|
+
if (hit && !t.journal.entries().some(e => isEntry(e, JT.attention) && e.item.id === id))
|
|
353
340
|
await t.journal.append(JT.attention, { item: { id, rev: 1, kind: "budget", text: "Workflow budget reached", wid: t.wid } });
|
|
354
341
|
return hit;
|
|
355
342
|
}
|
|
356
343
|
async function retireAttention(journal, call) {
|
|
357
|
-
for (const
|
|
358
|
-
const item = e.item;
|
|
344
|
+
for (const { item } of attentionEntries(journal.entries())) {
|
|
359
345
|
if (item.kind === "finished" || item.id === `unknown:${call}` || item.id.startsWith("fence:"))
|
|
360
346
|
continue;
|
|
361
347
|
if (item.call === call && !journal.entries().some(r => r.type === JT.attentionResolved && r.id === item.id && r.rev === item.rev))
|
|
@@ -365,7 +351,7 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
365
351
|
/** P15, AC4: A call sealed `unknown` raises exactly one unknown item for its origin; seal and recovery both run this. */
|
|
366
352
|
async function unknownAttention(journal, call) {
|
|
367
353
|
const result = sealed(journal, call), id = `unknown:${call}`;
|
|
368
|
-
if (result?.status !== "unknown" || journal.entries().some(e => e
|
|
354
|
+
if (result?.status !== "unknown" || journal.entries().some(e => isEntry(e, JT.attention) && e.item.id === id))
|
|
369
355
|
return;
|
|
370
356
|
const text = `Call ${call} ended with an unknown outcome: ${result.error || "no evidence of what it did"}. It was not re-run; check its effects before continuing.`;
|
|
371
357
|
await journal.append(JT.attention, { item: { id, rev: 1, kind: "unknown", text, wid: address(call).wid, call } });
|
|
@@ -454,15 +440,15 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
454
440
|
continue;
|
|
455
441
|
const probe = provider !== undefined && folded().exhausted.has(provider);
|
|
456
442
|
const holders = holdings().filter(e => e.pool === provider);
|
|
457
|
-
const limit =
|
|
443
|
+
const limit = settings().providers?.[provider ?? ""]?.slots ?? Infinity;
|
|
458
444
|
if (!capacity({ kind: "provider", holders: holders.length, capacity: limit }))
|
|
459
445
|
continue;
|
|
460
446
|
// A burst of dispatches all read the same MemAvailable before any child has grown: subtract the children
|
|
461
447
|
// admitted in the last 30 s (their memory is not visible yet), so a burst cannot over-commit the headroom.
|
|
462
|
-
const perChild =
|
|
448
|
+
const perChild = settings().memory?.perChildMb ?? 300;
|
|
463
449
|
const warming = holdings().filter(e => e.pool === "memory" && Date.now() - e.ts < MEM_WARMUP_MS).length;
|
|
464
450
|
const measured = await (options.memory ?? availableMemory)(), available = measured - warming * perChild;
|
|
465
|
-
const admitted = capacity({ kind: "memory", available, reserve:
|
|
451
|
+
const admitted = capacity({ kind: "memory", available, reserve: settings().memory?.reserveMb ?? 2048, perChild });
|
|
466
452
|
// F3: every admitted dispatch is recorded; repeated refusals at most once per 30 s per call.
|
|
467
453
|
if (admitted || a.refusedAt === undefined || Date.now() - a.refusedAt >= MEM_RECORD_MS) {
|
|
468
454
|
await orch.append("mem", { available, admitted, exec, ...(warming ? { measured, warming } : {}) });
|
|
@@ -508,9 +494,10 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
508
494
|
const recorded = sessionModel(entries);
|
|
509
495
|
const ownSegment = entries.some(e => e.type === "custom" && e.customType === CT.exec && typeof e.data?.exec === "string" && e.data.exec.startsWith(`${t.callId}#`));
|
|
510
496
|
const freshFork = t.spec.context === "fork" && !t.continueFrom && !ownSegment;
|
|
511
|
-
const
|
|
512
|
-
const
|
|
513
|
-
const
|
|
497
|
+
const { defaultModel: configured, pools } = settings();
|
|
498
|
+
const raw = t.spec.model ?? t.agent.model ?? configured ?? await defaultModel();
|
|
499
|
+
const pool = raw && pools?.[raw] ? raw : undefined;
|
|
500
|
+
const candidates = raw ? resolveModel(raw, pools) : [{ id: "" }];
|
|
514
501
|
const candidate = recorded && candidates.some(m => m.provider === recorded.provider && m.id === recorded.id);
|
|
515
502
|
// Leave the session's model for the pool's others when its pool skips it after losses, or its provider's usage
|
|
516
503
|
// window is used up; and at a new generation, go back to the pool's order of preference.
|
|
@@ -537,7 +524,7 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
537
524
|
if (typeof qid !== "string" || typeof rev !== "number")
|
|
538
525
|
continue;
|
|
539
526
|
const id = `q:${t.callId}:${qid}`;
|
|
540
|
-
if (!t.journal.entries().some(r => r
|
|
527
|
+
if (!t.journal.entries().some(r => isEntry(r, JT.attention) && r.item.id === id && r.item.rev === rev))
|
|
541
528
|
await t.journal.append(JT.attention, { item: { id, rev, kind: "question", text: String(question), wid: t.wid, call: t.callId, qid, session: callSession(home, t.wid, t.key, t.gen) } });
|
|
542
529
|
const answered = entries.some(r => {
|
|
543
530
|
const details = r.message?.details ?? r.details;
|
|
@@ -783,7 +770,7 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
783
770
|
if (!q || interrupted(a) || segment < 0 || !entries.slice(segment + 1).some(e => e.customType === CT.question && e.data?.qid === q.qid) || journal.entries().some(e => e.type === "answer-bound" && e.call === t.callId && e.qid === q.qid && e.rev === q.rev))
|
|
784
771
|
return;
|
|
785
772
|
await serial(async () => {
|
|
786
|
-
const attention = journal.entries().find(e => e
|
|
773
|
+
const attention = journal.entries().find(e => isEntry(e, JT.attention) && e.item.qid === q.qid && e.item.call === t.callId && e.item.rev === q.rev);
|
|
787
774
|
if (attention && Date.now() - attention.ts >= (config.k?.hibernateMs ?? 120000) && !has(journal, JT.fenced, exec) && hibernation(journal, t.callId)?.exec !== exec) {
|
|
788
775
|
await journal.append("hibernated", { call: t.callId, exec, qid: q.qid, rev: q.rev });
|
|
789
776
|
a.wake();
|
|
@@ -870,7 +857,7 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
870
857
|
if (sleeping.qid !== req.cond?.qid || sleeping.rev !== req.cond?.rev)
|
|
871
858
|
return { action: "reject", reason: "stale-rev" };
|
|
872
859
|
const rid2 = forwardRid(req.rid, ctx.widRev, ctx.key, hash);
|
|
873
|
-
const item = ctx.journal.entries().find(e => e.
|
|
860
|
+
const item = attentionEntries(ctx.journal.entries()).find(e => e.item.call === dest && e.item.qid === sleeping.qid && e.item.rev === sleeping.rev)?.item;
|
|
874
861
|
await ctx.journal.append("answer-bound", { call: dest, qid: sleeping.qid, rev: sleeping.rev, rid: req.rid, rid2, hash, message: `Question: ${item?.text ?? sleeping.qid}\nAnswer: ${req.body.message ?? ""}` });
|
|
875
862
|
active.get(dest)?.wake();
|
|
876
863
|
wake();
|
|
@@ -904,7 +891,7 @@ export default function createExecutor(ledgers, options = {}) {
|
|
|
904
891
|
const held = holdings().filter(h => h.exec === exec);
|
|
905
892
|
if (!held.some(h => h.pool === provider)) {
|
|
906
893
|
const target = holdings().filter(h => h.pool === provider);
|
|
907
|
-
if (!capacity({ kind: "provider", holders: target.length, capacity:
|
|
894
|
+
if (!capacity({ kind: "provider", holders: target.length, capacity: settings().providers?.[provider]?.slots ?? Infinity }))
|
|
908
895
|
return { action: "reject", reason: "provider-full" };
|
|
909
896
|
let slot = 0;
|
|
910
897
|
while (target.some(h => h.slot === slot))
|
|
@@ -5,7 +5,7 @@ import { join } from "node:path";
|
|
|
5
5
|
import { stat as fileStat } from "node:fs/promises";
|
|
6
6
|
import { performance } from "node:perf_hooks";
|
|
7
7
|
import { createInterface } from "node:readline";
|
|
8
|
-
import { JT } from "../../types.js";
|
|
8
|
+
import { JT, attentionEntries } from "../../types.js";
|
|
9
9
|
import { callDir, callSession } from "../../paths.js";
|
|
10
10
|
import { monotoneTime } from "../../kernel/guards.js";
|
|
11
11
|
import { readSession } from "./session.js";
|
|
@@ -42,8 +42,7 @@ export async function observeExecution(d) {
|
|
|
42
42
|
};
|
|
43
43
|
const stall = () => serial(async () => {
|
|
44
44
|
const id = `stall:${t.callId}`;
|
|
45
|
-
const items = t.journal.entries()
|
|
46
|
-
const last = items.at(-1)?.item;
|
|
45
|
+
const items = attentionEntries(t.journal.entries(), id), last = items.at(-1)?.item;
|
|
47
46
|
const open = last && !t.journal.entries().some(e => e.type === JT.attentionResolved && e.id === id && e.rev === last.rev);
|
|
48
47
|
const fresh = items.at(-1)?.exec === exec ? clock.last > Number(items.at(-1)?.horizon) : clock.last > started;
|
|
49
48
|
if (open && fresh)
|
|
@@ -53,8 +52,7 @@ export async function observeExecution(d) {
|
|
|
53
52
|
});
|
|
54
53
|
const noProgress = () => serial(async () => {
|
|
55
54
|
const id = `noprogress:${t.callId}`;
|
|
56
|
-
const items = t.journal.entries()
|
|
57
|
-
const last = items.at(-1)?.item;
|
|
55
|
+
const items = attentionEntries(t.journal.entries(), id), last = items.at(-1)?.item;
|
|
58
56
|
const open = last && !t.journal.entries().some(e => e.type === JT.attentionResolved && e.id === id && e.rev === last.rev);
|
|
59
57
|
const fresh = items.at(-1)?.exec === exec ? progress > Number(items.at(-1)?.horizon) : progress > started;
|
|
60
58
|
if (open && fresh)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { JT } from "../../types.js";
|
|
1
|
+
import { JT, attentionEntries, isEntry } from "../../types.js";
|
|
2
2
|
import { Containment as PlatformContainment } from "../../platform/containment.js";
|
|
3
3
|
import { ProcessTable } from "../../platform/proctable.js";
|
|
4
4
|
/** C2, F4: A containment whose process-table snapshots never overlap (ProcessTable.list is not reentrant). */
|
|
@@ -21,12 +21,27 @@ export async function skipLostCandidate(journal, orch, exec) {
|
|
|
21
21
|
if (tail.length === 3 && tail.every(e => e.type === "candidate-loss" && e.model === name))
|
|
22
22
|
await orch.append("skip", { pool: selected.pool, model: name, until: Date.now() + 600000 });
|
|
23
23
|
}
|
|
24
|
+
// The gate path records outside the executor's serial section and the sweep inside it: each check-then-append of a
|
|
25
|
+
// fence record runs in this per-journal section, so two recorders that both find no record still write it once (A5).
|
|
26
|
+
const sections = new WeakMap();
|
|
27
|
+
/** A5: Run a check-then-append of fence records alone on its journal. Not reentrant: never nest two. */
|
|
28
|
+
export function recordOnce(journal, operation) {
|
|
29
|
+
const result = (sections.get(journal) ?? Promise.resolve()).then(operation);
|
|
30
|
+
const tail = result.catch(() => { });
|
|
31
|
+
sections.set(journal, tail);
|
|
32
|
+
void tail.then(() => { if (sections.get(journal) === tail)
|
|
33
|
+
sections.delete(journal); });
|
|
34
|
+
return result;
|
|
35
|
+
}
|
|
24
36
|
/** F1: Record a fence timeout of an execution or gate identity once, with one unknown attention item for the origin. */
|
|
25
|
-
export
|
|
37
|
+
export function recordFenceFailure(journal, id, call, error) {
|
|
38
|
+
return recordOnce(journal, () => fenceFailure(journal, id, call, error));
|
|
39
|
+
}
|
|
40
|
+
async function fenceFailure(journal, id, call, error) {
|
|
26
41
|
if (!journal.entries().some(e => e.type === "fence-failed" && e.exec === id))
|
|
27
42
|
await journal.append("fence-failed", { exec: id, error: String(error) });
|
|
28
43
|
const item = `fence:${id}`;
|
|
29
|
-
if (journal.entries().some(e => e
|
|
44
|
+
if (journal.entries().some(e => isEntry(e, JT.attention) && e.item.id === item))
|
|
30
45
|
return;
|
|
31
46
|
const text = id.startsWith("gate:")
|
|
32
47
|
? `Processes of gate ${id} did not exit after SIGKILL (${String(error)}). Its outcome stays unknown and is recorded once they are gone; check for stuck processes (e.g. blocked I/O).`
|
|
@@ -34,8 +49,12 @@ export async function recordFenceFailure(journal, id, call, error) {
|
|
|
34
49
|
await journal.append(JT.attention, { item: { id: item, rev: 1, kind: "unknown", text, wid: call.slice(0, call.lastIndexOf("@", call.indexOf("/"))), call } });
|
|
35
50
|
}
|
|
36
51
|
/** F1: Resolve the fence attention item of an identity that a later fence proved retired. */
|
|
37
|
-
export
|
|
38
|
-
|
|
52
|
+
export function resolveFenceAttention(journal, id) {
|
|
53
|
+
return recordOnce(journal, () => fenceAttentionResolved(journal, id));
|
|
54
|
+
}
|
|
55
|
+
/** The same, inside a `recordOnce` section. */
|
|
56
|
+
export async function fenceAttentionResolved(journal, id) {
|
|
57
|
+
const item = attentionEntries(journal.entries(), `fence:${id}`)[0]?.item;
|
|
39
58
|
if (item && !journal.entries().some(e => e.type === JT.attentionResolved && e.id === `fence:${id}` && e.rev === item.rev))
|
|
40
59
|
await journal.append(JT.attentionResolved, { id: `fence:${id}`, rev: item.rev, resolution: "fenced" });
|
|
41
60
|
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// A1, A4: the state of the orchestrator ledger that admission reads and `status` shows, folded by one reducer.
|
|
2
|
+
// hold/release{pool,slot,exec}: provider and memory slots held (a release frees only its holder's slot).
|
|
3
|
+
// switch-observed{exec,rid}: a model switch the execution's provider requests confirmed.
|
|
4
|
+
// skip{pool,model,until}: a pool candidate skipped after repeated losses (K7).
|
|
5
|
+
// provider-*: used-up usage windows (providers.ts).
|
|
6
|
+
// config{hash,config}: the orchestrator settings in effect from here; config-rejected{error,hash?}: a change of
|
|
7
|
+
// config.json refused, while the earlier settings stay (cleared by the next config record).
|
|
8
|
+
import { foldExhaustion } from "./providers.js";
|
|
9
|
+
import { isEntry } from "../types.js";
|
|
10
|
+
export function emptyLedger() {
|
|
11
|
+
return { seen: 0, held: new Map(), observed: new Set(), skips: new Map(), exhausted: new Map() };
|
|
12
|
+
}
|
|
13
|
+
/** Apply one orchestrator ledger entry. */
|
|
14
|
+
export function applyLedger(state, e) {
|
|
15
|
+
if (isEntry(e, "hold"))
|
|
16
|
+
state.held.set(`${e.pool}:${e.slot}`, e);
|
|
17
|
+
else if (isEntry(e, "release")) {
|
|
18
|
+
const id = `${e.pool}:${e.slot}`;
|
|
19
|
+
if (state.held.get(id)?.exec === e.exec)
|
|
20
|
+
state.held.delete(id);
|
|
21
|
+
}
|
|
22
|
+
else if (isEntry(e, "switch-observed"))
|
|
23
|
+
state.observed.add(`${e.exec}\n${e.rid}`);
|
|
24
|
+
else if (isEntry(e, "skip"))
|
|
25
|
+
state.skips.set(`${e.pool}\n${e.model}`, Math.max(Number(e.until), state.skips.get(`${e.pool}\n${e.model}`) ?? 0));
|
|
26
|
+
else if (isEntry(e, "config")) {
|
|
27
|
+
state.config = { hash: String(e.hash), settings: e.config, ts: e.ts };
|
|
28
|
+
delete state.rejected;
|
|
29
|
+
}
|
|
30
|
+
else if (isEntry(e, "config-rejected"))
|
|
31
|
+
state.rejected = { error: String(e.error), ts: e.ts };
|
|
32
|
+
foldExhaustion(state.exhausted, e);
|
|
33
|
+
}
|
|
34
|
+
/** Fold the entries appended since `state` was last folded (the ledger only grows); a ledger that is not the one
|
|
35
|
+
* folded so far (shorter, or another entry where the last folded one was) is folded from the start. */
|
|
36
|
+
export function foldLedger(state, entries) {
|
|
37
|
+
// Journal readers keep the entry objects of a ledger as it grows (kernel/journal.ts), so identity tells them apart.
|
|
38
|
+
if (state.seen && entries[state.seen - 1] !== state.last)
|
|
39
|
+
Object.assign(state, emptyLedger(), { config: undefined, rejected: undefined, last: undefined });
|
|
40
|
+
for (; state.seen < entries.length; state.seen++)
|
|
41
|
+
applyLedger(state, entries[state.seen]);
|
|
42
|
+
state.last = entries[state.seen - 1];
|
|
43
|
+
return state;
|
|
44
|
+
}
|
|
45
|
+
/** The provider slots held now (memory slots included). */
|
|
46
|
+
export const holdings = (state) => [...state.held.values()];
|
|
47
|
+
/** A4: the settings guards read: the ones recorded last, else (a ledger that records none: an embedded or test
|
|
48
|
+
* orchestrator given its settings) the given ones. */
|
|
49
|
+
export const settingsOf = (state, given) => state.config?.settings ?? given;
|
|
@@ -39,8 +39,9 @@ export async function main(options = {}) {
|
|
|
39
39
|
const factory = options.executor ?? (await import(__rewriteRelativeImportExtension(new URL(import.meta.url.endsWith('.ts') ? './executor/index.ts' : './executor/index.js', import.meta.url).href))).default;
|
|
40
40
|
const executor = factory(ledgers);
|
|
41
41
|
engine = new Engine(ledgers, executor, options);
|
|
42
|
+
// A4: admission reads the settings recorded last, so the ones this run starts with are recorded (given ones too).
|
|
43
|
+
await recordConfig(ledgers.orch, config);
|
|
42
44
|
if (stamp !== undefined) {
|
|
43
|
-
await recordConfig(ledgers.orch, config);
|
|
44
45
|
// A change applies between slot admissions, so one admission never mixes two versions of the limits.
|
|
45
46
|
watcher = watchConfig({ path: configPath(home), stamp, config, orch: ledgers.orch, intervalMs: Math.min(1000, config.k?.trackerMs ?? 1000),
|
|
46
47
|
apply: change => executor.reconfigure ? executor.reconfigure(change) : change() });
|
|
@@ -5,8 +5,8 @@ import { join } from "node:path";
|
|
|
5
5
|
import { compileFanout } from "../compat/fanout.js";
|
|
6
6
|
import { readJournalSnapshot } from "../kernel/journal.js";
|
|
7
7
|
import { journalPath, orchLedger, pinnedDir, workflowDir } from "../paths.js";
|
|
8
|
-
import { JT } from "../types.js";
|
|
9
|
-
import {
|
|
8
|
+
import { JT, isEntry } from "../types.js";
|
|
9
|
+
import { emptyLedger, foldLedger } from "./ledger.js";
|
|
10
10
|
/** A workflow has live work: it runs, or follow-ups opened on it after it finished have not ended yet. */
|
|
11
11
|
export function isLive(wf) {
|
|
12
12
|
return wf.status === "running" || (wf.followUps ?? 0) > 0;
|
|
@@ -191,7 +191,7 @@ function snapshotReducer(wid, entries) {
|
|
|
191
191
|
delete call.hibernated;
|
|
192
192
|
hibernating.delete(call.callId);
|
|
193
193
|
}
|
|
194
|
-
else if (e
|
|
194
|
+
else if (isEntry(e, JT.attention)) {
|
|
195
195
|
const item = e.item;
|
|
196
196
|
attention.push(item);
|
|
197
197
|
}
|
|
@@ -469,34 +469,24 @@ const tokens = (u) => { const n = u ? u.input + u.output : 0; return n >= 1e6 ?
|
|
|
469
469
|
const latestCalls = (wf) => [...new Map(wf.calls.map(c => [c.key, c])).values()];
|
|
470
470
|
/** Provider slots from the orchestrator ledger: holders per provider (hold/release{pool,slot,exec}) and the limits of
|
|
471
471
|
* the settings in effect (the latest config{hash,config}); config-rejected after it is reported too. */
|
|
472
|
+
// One fold per orchestrator ledger, extended as it grows (ledger.ts; the executor folds the same entries the same way).
|
|
473
|
+
const ledgerStates = new Map();
|
|
472
474
|
export function slotsView(home, now = Date.now()) {
|
|
473
|
-
const
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
if (e.type === "hold")
|
|
477
|
-
held.set(`${e.pool}:${e.slot}`, e);
|
|
478
|
-
else if (e.type === "release" && held.get(`${e.pool}:${e.slot}`)?.exec === e.exec)
|
|
479
|
-
held.delete(`${e.pool}:${e.slot}`);
|
|
480
|
-
else if (e.type === "config") {
|
|
481
|
-
config = e;
|
|
482
|
-
rejected = undefined;
|
|
483
|
-
}
|
|
484
|
-
else if (e.type === "config-rejected")
|
|
485
|
-
rejected = e;
|
|
486
|
-
foldExhaustion(used, e);
|
|
487
|
-
}
|
|
475
|
+
const path = orchLedger(home), state = foldLedger(ledgerStates.get(path) ?? emptyLedger(), readJournalSnapshot(path));
|
|
476
|
+
ledgerStates.set(path, state);
|
|
477
|
+
const { held, config, rejected } = state, used = state.exhausted;
|
|
488
478
|
const exhausted = [...used].sort(([a], [b]) => a.localeCompare(b)).map(([p, x]) => `${p} exhausted since ${age(now - x.since)} ago (${clip(x.error, 80)}), ` +
|
|
489
479
|
(x.probe ? `probing with ${x.probe.split("#")[0]}` : x.nextTry > now ? `next try in ${age(x.nextTry - now)}` : "next call probes it"));
|
|
490
|
-
const limits =
|
|
480
|
+
const limits = config?.settings.providers ?? {};
|
|
491
481
|
const holders = new Map();
|
|
492
482
|
for (const e of held.values())
|
|
493
483
|
if (e.pool !== "memory")
|
|
494
|
-
holders.set(
|
|
484
|
+
holders.set(e.pool, (holders.get(e.pool) ?? 0) + 1);
|
|
495
485
|
const names = [...new Set([...Object.keys(limits), ...holders.keys()])].sort();
|
|
496
486
|
const slots = names.map(p => { const n = holders.get(p) ?? 0, limit = limits[p]?.slots; return typeof limit === "number" ? `${p} ${n}/${limit}` : `${p} ${n} (no limit)`; });
|
|
497
|
-
return { ...(slots.length ? { slots } : {}), ...(config ? { config: `${
|
|
487
|
+
return { ...(slots.length ? { slots } : {}), ...(config ? { config: `${config.hash} since ${age(now - config.ts)} ago` } : {}),
|
|
498
488
|
...(exhausted.length ? { exhausted } : {}),
|
|
499
|
-
...(rejected ? { configRejected: `${clip(
|
|
489
|
+
...(rejected ? { configRejected: `${clip(rejected.error, 200)} (${age(now - rejected.ts)} ago); ${config ? config.hash : "the start settings"} stay in effect` } : {}) };
|
|
500
490
|
}
|
|
501
491
|
/** Tool status without a wid: what runs, what waits for an answer and what failed, with finished workflows one line each.
|
|
502
492
|
* The full view (every call's last line and usage) ran to tens of thousands of tokens on a busy home. */
|
package/dist/types.js
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
// Pinned shared contracts for pi-durable-subagents.
|
|
2
2
|
// Changing anything here needs parent approval: every module and test depends on it.
|
|
3
3
|
// Design references (repo-local, not published): design/principles.md (A1-A5, V1-V8), design/spec.md (P1-P38, K1-K11).
|
|
4
|
+
/** Narrow an entry to its type's fields (their shape is the writer's contract, not checked at runtime). */
|
|
5
|
+
export function isEntry(e, type) { return e.type === type; }
|
|
6
|
+
/** P15: the attention entries of a journal, of one item id when given (each revision of an item is an entry). */
|
|
7
|
+
export function attentionEntries(entries, id) {
|
|
8
|
+
return entries.filter((e) => isEntry(e, "attention") && (id === undefined || e.item.id === id));
|
|
9
|
+
}
|
|
4
10
|
// ---------------------------------------------------------------------------
|
|
5
11
|
// Child session entries (pi session file is the child's log; P4, P8, P23, P24)
|
|
6
12
|
// customType values written by the session agent. All carry details/data as below.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-durable-subagents",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.12",
|
|
4
4
|
"description": "Subagents for pi that never lose work and never do it twice. Crash-safe workflows, automatic recovery, and a live view just like the main agent.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|