pi-durable-subagents 1.0.16 → 1.0.18

package/dist/cli/main.js

@@ -12,7 +12,9 @@ import { resolution, start, startOrchestrator, submit } from "./control.js";
 import { doctor, renderDoctor, size } from "./doctor.js";
 import { smoke } from "./smoke.js";
 import { serviceFiles, manageService } from "./service.js";
-const commands = ["smoke", "tail", "status", "events", "start", "resume", "drain", "stop", "stop-all", "prune", "doctor", "install-service", "uninstall-service", "help"];
+import { leaseLines, leaseState } from "../platform/lease.js";
+import { currentOrchestrator, legacyRestart, waitExit, waitSuccessor } from "./restart.js";
+const commands = ["smoke", "tail", "status", "events", "start", "resume", "drain", "stop", "stop-all", "prune", "restart", "leases", "doctor", "install-service", "uninstall-service", "help"];
 /** P25: Reject ambiguous CLI arguments before any durable action. */
 export function parseArgs(args) {
     if (!args.length || (args.length === 1 && ["-h", "--help"].includes(args[0])))
@@ -31,11 +33,17 @@ export function parseArgs(args) {
         olderThanDays = Number(value);
         rest = [...rest.slice(0, at), ...rest.slice(at + 2)];
     }
+    const force = rest.includes("--force");
+    if (force && command !== "restart")
+        throw new Error("--force is only supported by restart");
+    if (rest.filter(a => a === "--force").length > 1)
+        throw new Error("Unknown or repeated option");
+    rest = rest.filter(a => a !== "--force");
     const json = rest.includes("--json"), dryRun = rest.includes("--dry-run");
     if (dryRun && !["install-service", "uninstall-service"].includes(command))
         throw new Error("--dry-run requires a service command");
-    if (json && !["status", "events", "tail", "doctor"].includes(command))
-        throw new Error("--json is only supported by status, events, tail and doctor");
+    if (json && !["status", "events", "tail", "doctor", "leases"].includes(command))
+        throw new Error("--json is only supported by status, events, tail, leases and doctor");
     if (rest.filter(a => a === "--json").length > 1 || rest.filter(a => a === "--dry-run").length > 1 || rest.some(a => a.startsWith("-") && a !== "--json" && a !== "--dry-run"))
         throw new Error("Unknown or repeated option");
     const targets = rest.filter(a => a !== "--json" && a !== "--dry-run");
@@ -48,7 +56,7 @@ export function parseArgs(args) {
         throw new Error("Invalid workflow id");
     if (target && olderThanDays !== undefined)
         throw new Error("prune takes a workflow id or --older-than, not both");
-    return { command, json, ...(dryRun ? { dryRun } : {}), ...(target ? { target } : {}), ...(olderThanDays !== undefined ? { olderThanDays } : {}) };
+    return { command, json, ...(force ? { force } : {}), ...(dryRun ? { dryRun } : {}), ...(target ? { target } : {}), ...(olderThanDays !== undefined ? { olderThanDays } : {}) };
 }
 const clip = (text, n) => text.length > n ? `${text.slice(0, n)}…` : text;
 /** P25, T10: Render journal-derived status (one line per call, last output line only) without live orchestrator memory. */
@@ -57,7 +65,7 @@ export function renderStatus(wf) {
     return [`${wf.wid}@${wf.rev}${wf.name ? ` ${wf.name}` : ""}: ${wf.status}${wf.error ? ` (${clip(wf.error, 300)})` : ""} · ${sealed}/${total} sealed${used ? ` · ${formatUsage(wf.usage)}` : ""}`,
         ...wf.calls.map(c => {
             const last = c.result?.output?.split("\n").map(l => l.trim()).filter(Boolean).at(-1), err = c.result?.error;
-            return `  ${c.key}@${c.gen} ${c.result?.status ?? c.phase}${c.model ? ` ${c.model}` : ""}${c.tools ? ` tools:${c.tools}` : ""}${c.usage && (c.usage.input || c.usage.output) ? ` ${formatUsage(c.usage)}` : ""}${last ? ` ${JSON.stringify(clip(last, 160))}` : err ? ` (${clip(err, 160)})` : ""}${c.sharedWorktree ? ` (shares worktree with ${c.sharedWorktree.join(", ")})` : ""}`;
+            return `  ${c.key}@${c.gen} ${c.result?.status ?? c.phase}${c.model ? ` ${c.model}` : ""}${c.tools ? ` tools:${c.tools}` : ""}${c.usage && (c.usage.input || c.usage.output) ? ` ${formatUsage(c.usage)}` : ""}${last ? ` ${JSON.stringify(clip(last, 160))}` : err ? ` (${clip(err, 160)})` : ""}${c.sharedWorktree ? ` (shares worktree with ${c.sharedWorktree.join(", ")})` : ""}${c.writerWait && !c.result && c.phase !== "sealed" ? ` (waiting for writer lock: ${c.writerWait.root} held by ${c.writerWait.holder})` : ""}`;
         }),
         ...wf.attention.map(a => `  ${a.kind}: ${JSON.stringify(clip(a.text, 300))}`),
         ...(wf.scriptLog ? [`  script log: ${wf.scriptLog}`] : [])].join("\n");
@@ -65,13 +73,13 @@ export function renderStatus(wf) {
 /** P25, T10: Render the compact status projection shared with the `subagents` tool. */
 export function renderView(view) {
     const lines = view.workflows.map(w => [`${w.wid}@${w.rev}${w.name ? ` ${w.name}` : ""}: ${w.status}${w.followUps ? " (follow-up running)" : ""}${w.error ? ` (${clip(w.error, 200)})` : ""} · ${w.done}/${w.planned ?? w.calls.length}${w.planned === undefined && w.status === "running" ? "+" : ""} done${w.usage.input || w.usage.output || w.usage.costUsd ? ` · ${formatUsage(w.usage)}` : ""}`,
-        ...w.calls.map(c => `  ${c.key}@${c.gen} ${c.status ?? c.phase}${c.hibernated ? " (hibernated, no slot)" : ""}${c.model ? ` ${c.model}` : ""}${c.switching ? ` → ${c.switching} (requested)` : ""}${c.switchFailed ? ` (switch refused: ${c.switchFailed})` : ""}${c.tools ? ` tools:${c.tools}` : ""}${c.usage ? ` ${formatUsage(c.usage)}` : ""}${c.lastLine ? ` ${JSON.stringify(c.lastLine)}` : c.error ? ` (${c.error})` : ""}${c.sharedWorktree ? ` (shares worktree with ${c.sharedWorktree.join(", ")})` : ""}`),
+        ...w.calls.map(c => `  ${c.key}@${c.gen} ${c.status ?? c.phase}${c.hibernated ? " (hibernated, no slot)" : ""}${c.model ? ` ${c.model}` : ""}${c.switching ? ` → ${c.switching} (requested)` : ""}${c.switchFailed ? ` (switch refused: ${c.switchFailed})` : ""}${c.tools ? ` tools:${c.tools}` : ""}${c.usage ? ` ${formatUsage(c.usage)}` : ""}${c.lastLine ? ` ${JSON.stringify(c.lastLine)}` : c.error ? ` (${c.error})` : ""}${c.sharedWorktree ? ` (shares worktree with ${c.sharedWorktree.join(", ")})` : ""}${c.writerWait && c.phase !== "sealed" ? ` (waiting for writer lock: ${c.writerWait.root} held by ${c.writerWait.holder})` : ""}${c.lease && c.phase !== "sealed" ? ` (${c.lease})` : ""}`),
         ...w.attention.map(a => `  ${a.kind}: ${JSON.stringify(a.text.split("\n")[0])}`)].join("\n"));
     if (view.paused)
         lines.unshift(`${view.paused} (pi-durable-subagents resume)`);
     if (view.olderFinished)
         lines.push(`(+${view.olderFinished} older finished workflows; status <wid> shows one in detail)`);
-    const footer = [view.slots?.length ? `slots: ${view.slots.join(", ")}` : "", view.config ? `config: ${view.config}` : "", view.configRejected ? `config.json rejected: ${view.configRejected}` : "", view.orchestrator ? `orchestrator: ${view.orchestrator}` : "", ...(view.exhausted ?? []), view.versionNote ? `note: ${view.versionNote}` : ""].filter(Boolean);
+    const footer = [view.slots?.length ? `slots: ${view.slots.join(", ")}` : "", view.config ? `config: ${view.config}` : "", view.configRejected ? `config.json rejected: ${view.configRejected}` : "", view.orchestrator ? `orchestrator: ${view.orchestrator}` : "", ...(view.leases ?? []).map(l => `lease: ${l}`), ...(view.exhausted ?? []), view.versionNote ? `note: ${view.versionNote}` : ""].filter(Boolean);
     if (!lines.length)
         lines.push("No workflows");
     return [...lines, ...footer].join("\n");
@@ -110,12 +118,53 @@ export function serviceEntryError(entry) {
         return `install-service refuses to run from an npx cache (${entry}); the cache can be pruned and the service would break. Install the CLI with \`npm i -g pi-durable-subagents\` and run \`pi-durable-subagents install-service\` again.`;
     return undefined;
 }
-export const HELP = "pi-durable-subagents: smoke | status [wid] [--json] | events <wid> [--json] | tail [wid] [--json] | start | resume [wid] | drain | stop <wid|callId> | stop-all | prune [wid] [--older-than <days>] | doctor [--json] | install-service [--dry-run] | uninstall-service [--dry-run] | chaos [--scenario <1-9>] [--keep] [--json]";
+export const HELP = "pi-durable-subagents: smoke | status [wid] [--json] | events <wid> [--json] | tail [wid] [--json] | start | resume [wid] | drain | stop <wid|callId> | stop-all | prune [wid] [--older-than <days>] | restart [--force] | hold <resource> [--shared] [--max-wait <s>] [--note <text>] -- <command…> | leases [--json] | doctor [--json] | install-service [--dry-run] | uninstall-service [--dry-run] | chaos [--scenario <1-9>] [--keep] [--json]";
+/** Restart: the orchestrator exits when no execution runs (or `force`) and the installed version takes over. */
+async function restartCommand(home, env, write, options) {
+    const starter = options.starter ?? startOrchestrator, old = currentOrchestrator(home);
+    if (!old) {
+        write(await start(home, env, starter) ? "restart: no orchestrator was running; started one for the pending work" : "restart: no orchestrator is running (one starts when work is submitted)");
+        return 0;
+    }
+    const waitMs = options.waitMs ?? 30_000;
+    if (!old.restart) {
+        // An orchestrator older than the restart request: check its journals here and end it with SIGTERM.
+        const legacy = legacyRestart(home, old, options.force);
+        if (!legacy.applied) {
+            write(`restart refused: ${legacy.reason}`);
+            return 1;
+        }
+    }
+    else {
+        const [req] = await submit(home, "restart", undefined, env, { force: options.force });
+        const outcome = await resolution(home, req.rid, waitMs);
+        if (!outcome) {
+            write(`submitted restart ${req.rid}; not resolved within ${Math.round(waitMs / 1000)} s (see: pi-durable-subagents doctor)`);
+            return 1;
+        }
+        if (outcome.type === "rejected") {
+            write(`restart refused: ${outcome.reason}`);
+            return 1;
+        }
+    }
+    if (!await waitExit(old, Math.max(waitMs, 60_000))) {
+        write(`restart: orchestrator ${old.version} (pid ${old.pid}) is still shutting down; see: pi-durable-subagents status`);
+        return 1;
+    }
+    await starter(home, env);
+    const next = await waitSuccessor(home, old, 15_000);
+    write(next ? `restarted: orchestrator ${old.version} (pid ${old.pid}) → ${next.version} (pid ${next.pid})` : `restart: orchestrator ${old.version} (pid ${old.pid}) exited; the next one starts when work is pending`);
+    return 0;
+}
 /** P1, P21, P25, P38: Dispatch the public CLI using durable requests and read-only snapshots. */
 export async function main(args = process.argv.slice(2), options = {}) {
+    if (args[0] === "hold") {
+        const { hold, parseHold } = await import("./hold.js");
+        return hold(parseHold(args.slice(1)), { env: options.env ?? process.env });
+    }
     if (args[0] === "chaos")
         return (await import("./chaos/index.js")).chaos(args.slice(1), options.env ?? process.env, options.write);
-    const { command, target, json, dryRun, olderThanDays } = parseArgs(args), env = options.env ?? process.env;
+    const { command, target, json, dryRun, olderThanDays, force } = parseArgs(args), env = options.env ?? process.env;
     const home = dsaHome(env), write = options.write ?? (line => console.log(line));
     if (command === "help") {
         write(HELP);
@@ -201,6 +250,13 @@ export async function main(args = process.argv.slice(2), options = {}) {
             write(`  ${String(e.wid)}`);
         return 0;
     }
+    if (command === "leases") {
+        const state = leaseState(home);
+        write(json ? JSON.stringify(state, null, 2) : state.length ? leaseLines(state).join("\n") : "No leases held or waited for");
+        return 0;
+    }
+    if (command === "restart")
+        return restartCommand(home, env, write, { force: force === true, starter: options.starter, waitMs: options.waitMs });
     for (const req of await submit(home, command, target, env))
         write(`submitted ${req.kind} ${req.rid}`);
     return 0;

package/dist/cli/restart.js

@@ -0,0 +1,71 @@
+// Restart: exit the running orchestrator so the installed version takes over, without interrupting running calls.
+// The orchestrator decides (request kind "restart", RestartBody): it stops launching executions, refuses while any runs
+// (unless forced), and otherwise exits and starts its successor. An orchestrator older than this request kind (its
+// `orchestrator` record has no `restart: true`) would keep the request as an invalid inbox file and never idle-exit, so it
+// is never sent one: the client checks the journals itself and ends it with SIGTERM (its shutdown fences what still runs,
+// and recovery resumes it) — a check without the orchestrator's launch gate, so a launch can slip in between.
+import { setTimeout as delay } from "node:timers/promises";
+import { readJournalSnapshot } from "../kernel/journal.js";
+import { journalPath, orchLedger } from "../paths.js";
+import { JT } from "../types.js";
+import { emptyLedger, foldLedger } from "../orchestrator/ledger.js";
+import { allWorkflows, liveCalls, processAlive } from "../orchestrator/snapshot.js";
+/** The orchestrator that runs now: the last `orchestrator` record without an exit, whose process lives. */
+export function currentOrchestrator(home) {
+    const o = foldLedger(emptyLedger(), readJournalSnapshot(orchLedger(home))).orchestrator;
+    return o && !o.exited && processAlive(o.pid, o.start) ? { version: o.version, pid: o.pid, ...(o.start ? { start: o.start } : {}), ts: o.ts, ...(o.restart ? { restart: true } : {}) } : undefined;
+}
+/** Calls whose execution runs per the journals: running, asking without having hibernated (its child still runs), or
+ *  running a gate (a gate-intent without its outcome; an interrupted gate is not run again). */
+export function journalLiveCalls(home, now = Date.now()) {
+    const age = (ms) => ms < 60_000 ? `${Math.max(0, Math.round(ms / 1000))}s` : `${Math.round(ms / 60_000)}m`;
+    return allWorkflows(home).flatMap(wf => {
+        const entries = readJournalSnapshot(journalPath(home, wf.wid));
+        const done = new Set(entries.filter(e => e.type === "gate").map(e => String(e.id)));
+        const sealed = new Set(entries.filter(e => e.type === JT.sealed).map(e => String(e.call)));
+        const gates = new Map(entries.filter(e => e.type === "gate-intent" && !done.has(String(e.id)) && !sealed.has(String(e.call))).map(e => [String(e.call), e.ts]));
+        const where = wf.origin ? ` from ${wf.origin}` : "";
+        const running = liveCalls(wf).filter(c => !gates.has(c.callId) && (c.phase === "running" || (c.phase === "asking" && c.exec !== undefined && !c.hibernated)))
+            .map(c => `${wf.wid}/${c.key}${c.startedAt ? ` ${age(now - c.startedAt)}` : ""}${where}`);
+        const gating = [...gates].map(([call, ts]) => `${wf.wid}/${call.slice(call.indexOf("/") + 1, call.lastIndexOf("@"))} gate ${age(now - ts)}${where}`);
+        return [...running, ...gating];
+    });
+}
+/** For an orchestrator without the restart request kind: refuse while calls run (unless forced), else SIGTERM it. */
+export function legacyRestart(home, old, force) {
+    const live = journalLiveCalls(home);
+    if (live.length && !force) {
+        const shown = live.slice(0, 8).join("; ") + (live.length > 8 ? `; +${live.length - 8} more` : "");
+        return { applied: false, reason: `busy: ${live.length} running call${live.length === 1 ? "" : "s"} on orchestrator ${old.version} (pid ${old.pid}): ${shown} — retry when they finish (or drain first), or force to fence them; they resume on the new orchestrator` };
+    }
+    try {
+        process.kill(old.pid, "SIGTERM");
+    }
+    catch (error) {
+        if (error.code !== "ESRCH")
+            throw error;
+    }
+    return { applied: true };
+}
+/** Wait (bounded) until the process ends; false when it still runs at the deadline. */
+export async function waitExit(old, timeoutMs) {
+    const deadline = performance.now() + timeoutMs;
+    while (processAlive(old.pid, old.start)) {
+        if (performance.now() >= deadline)
+            return false;
+        await delay(100);
+    }
+    return true;
+}
+/** Wait (bounded) for an orchestrator other than `old` to record its start; undefined at the deadline. */
+export async function waitSuccessor(home, old, timeoutMs) {
+    const deadline = performance.now() + timeoutMs;
+    for (;;) {
+        const now = currentOrchestrator(home);
+        if (now && (!old || now.pid !== old.pid || now.ts !== old.ts))
+            return now;
+        if (performance.now() >= deadline)
+            return undefined;
+        await delay(100);
+    }
+}

package/dist/compat/spec.js

@@ -1,5 +1,5 @@
 import { schemaProblems } from "../agent/child/schema.js";
-const fields = new Set(["agent", "task", "model", "cwd", "timeoutMs", "output", "schema", "gate", "isolation", "context", "budget", "once", "tools", "skills", "key"]);
+const fields = new Set(["agent", "task", "model", "cwd", "timeoutMs", "output", "schema", "gate", "isolation", "context", "budget", "once", "tools", "skills", "key", "writer"]);
 const isObject = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
 const positive = (v) => typeof v === "number" && Number.isFinite(v) && v > 0;
 const text = (v) => typeof v === "string" && v.trim() !== "";
@@ -67,6 +67,8 @@ export function validateCallSpec(spec, options = {}) {
         errors.push(...budget(s.budget));
     if (has("once") && typeof s.once !== "boolean")
         errors.push(`once must be a boolean${got(s.once)}`);
+    if (has("writer") && typeof s.writer !== "boolean")
+        errors.push(`writer must be a boolean${got(s.writer)}`);
     for (const k of ["tools", "skills"])
         if (has(k) && (!Array.isArray(s[k]) || !s[k].every(v => typeof v === "string")))
             errors.push(`${k} must be an array of strings${got(s[k])}`);

package/dist/kernel/mailbox.js

@@ -50,7 +50,7 @@ function isRequest(value) {
     const r = value;
     // A3, C11: request identities are short and path-safe, so every derived path (staging, receipts) is valid.
     return typeof r.rid === 'string' && /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(r.rid) && typeof r.from === 'string' && typeof r.to === 'string' && Number.isSafeInteger(r.sseq) && r.sseq > 0 &&
-        ['run', 'send', 'stop', 'revise', 'resume', 'drain', 'prune', 'task', 'steer', 'follow-up', 'answer', 'model', 'continue', 'withdraw', 'call', 'emit'].includes(r.kind) && Object.hasOwn(r, 'body');
+        ['run', 'send', 'stop', 'revise', 'resume', 'drain', 'prune', 'restart', 'task', 'steer', 'follow-up', 'answer', 'model', 'continue', 'withdraw', 'call', 'emit'].includes(r.kind) && Object.hasOwn(r, 'body');
 }
 /** P3: Scan immutable requests, reporting invalid files without admitting them. */
 export async function scanInbox(inboxDir, report = (path, error) => console.warn(path, error)) {

package/dist/orchestrator/config.js

@@ -8,7 +8,7 @@ import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { contentHash } from "../kernel/ids.js";
 /** The keys the orchestrator reads; config.json also holds pi-side settings (ui, onQuit) that it ignores. */
-const KEYS = ["defaultModel", "pools", "providers", "memory", "k"];
+const KEYS = ["defaultModel", "pools", "providers", "memory", "writerLock", "k"];
 const K = ["lossBound", "checkpointMs", "stallMs", "progressMs", "switchTimeoutMs", "idleExitMs", "trackerMs", "hibernateMs", "spawnBudget", "probeMs"];
 export const configPath = (home) => join(home, "config.json");
 /** The orchestrator's part of a parsed config.json. */
@@ -23,7 +23,9 @@ const count = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0;
 export function configProblem(raw) {
     if (!record(raw))
         return "config.json must be a JSON object";
-    const { defaultModel, pools, providers, memory, k } = raw;
+    const { defaultModel, pools, providers, memory, writerLock, k } = raw;
+    if (writerLock !== undefined && writerLock !== "queue" && writerLock !== "off")
+        return 'writerLock must be "queue" or "off"';
     if (defaultModel !== undefined && typeof defaultModel !== "string")
         return "defaultModel must be a string";
     if (pools !== undefined) {

package/dist/orchestrator/engine.js

@@ -91,6 +91,7 @@ export class Engine {
     queue = Promise.resolve();
     failure;
     closed = false;
+    restarting = false;
     generations = new Set();
     // wid -> executor runs whose follow-up has not run yet (prune never closes a journal they may still append to).
     running = new Map();
@@ -442,12 +443,36 @@ export class Engine {
                         st.running.clear();
             }
         }
+        else if (req.kind === 'restart') {
+            // A replay (the restart was recorded, then the process ended before its resolution) applies without restarting again.
+            if (!this.ledgers.orch.entries().some(e => e.type === 'restart' && e.rid === req.rid)) {
+                const force = req.body?.force === true;
+                const gate = this.executor.quiesce?.() ?? { live: [], resume() { } };
+                if (gate.live.length && !force) {
+                    gate.resume();
+                    return { action: 'reject', reason: this.busyReason(gate.live) };
+                }
+                await this.ledgers.orch.append('restart', { rid: req.rid, force, live: gate.live.map(l => l.exec) });
+                this.restarting = true;
+            }
+        }
         else if (req.kind === 'prune')
             return this.prune(req);
         else
             return { action: 'reject', reason: 'unsupported-kind' };
         return { action: 'apply' };
     }
+    /** Restart refused: which executions run now, whose they are and for how long, and the way to proceed. */
+    busyReason(live) {
+        const now = Date.now(), minutes = (ms) => ms < 60_000 ? `${Math.max(0, Math.round(ms / 1000))}s` : `${Math.round(ms / 60_000)}m`;
+        const shown = live.slice(0, 8).map(l => {
+            const origin = this.store.workflows.get(l.wid)?.origin;
+            return `${l.wid}/${l.key}${l.phase === 'gate' ? ' gate' : ''} ${minutes(now - l.since)}${origin ? ` from ${origin}` : ''}`;
+        });
+        return `busy: ${live.length} running execution${live.length === 1 ? '' : 's'}: ${shown.join('; ')}${live.length > shown.length ? `; +${live.length - shown.length} more` : ''} — retry when they finish (or drain first), or force to fence them; they resume on the new orchestrator`;
+    }
+    /** Set by an applied restart: the loop ends and the process exits; its successor recovers every workflow. */
+    get restartRequested() { return this.restarting; }
     /** A1, housekeeping: Prune finished workflows (named, or all ended more than olderThanDays ago); a replay of a
      *  committed prune is applied again and continues with the workflows still eligible. */
     async prune(req) {
@@ -729,7 +754,7 @@ export class Engine {
         this.poll = setInterval(() => this.background(() => this.consume()), 1000);
         let idleSince = performance.now();
         try {
-            while (!signal?.aborted && !this.closed) {
+            while (!signal?.aborted && !this.closed && !this.restarting) {
                 await this.queue;
                 if (this.failure)
                     throw this.failure;