pi-durable-subagents 1.0.22 → 1.0.24

package/dist/events/pump.js

@@ -0,0 +1,351 @@
+// R2: the orchestrator's event pump. Derives events from the durable sources (workflow journals, the orchestrator
+// ledger's `created`) after their appends and writes them to the event log with the watermarks of the sources derived
+// (same fsync), so a restart re-derives only after them (at least once). It is also the EventSink other producers
+// (R7 waiting/moving) emit through. One chain serializes passes, emits, compaction and close. A pass that cannot log
+// what it derived retries in the background, and `flush` (prune) rejects; a broken log (a failure after a compaction's
+// rename) is reopened by the next pass or emit.
+// Memory: per unpruned workflow one watermark and one cached identity; reads the journals' committed arrays from their
+// watermark (no journal copies per tick); a pass holds at most one chunk (EVENT_SEQ_SKIP drafts) at a time.
+import { eventsLog } from "../paths.js";
+import { JT, isEntry } from "../types.js";
+import { requestId } from "../requests.js";
+import { deriveCreated, deriveEntry, labelsOf } from "./derive.js";
+import { EventLog } from "./log.js";
+import { EVENT_RETENTION_MS, EVENT_SEQ_SKIP } from "./types.js";
+const COMPACT_EVERY_MS = 3600_000;
+const committed = (j) => j.committed?.() ?? j.entries();
+/** Retention: a workflow whose events may go once old: final (done/failed/stopped, not parked) in its current
+ *  revision, no open question and no call of that revision without a seal (or retirement). */
+export function quiet(wid, entries) {
+    let revision = 1, done;
+    const calls = new Set(), ended = new Set(), open = new Set();
+    for (const e of entries) {
+        if (e.type === "revised") {
+            revision = Number(e.revision);
+            calls.clear();
+            done = undefined;
+        }
+        else if (e.type === JT.done)
+            done = e;
+        else if (e.type === "resumed" && !e.call)
+            done = undefined;
+        else if (e.type === "call")
+            calls.add(`${wid}@${revision}/${String(e.key)}@${String(e.gen)}`);
+        else if (e.type === "generation" && Number(e.revision) === revision)
+            calls.add(`${wid}@${revision}/${String(e.key)}@${String(e.gen)}`);
+        else if (e.type === JT.sealed || e.type === "retired")
+            ended.add(String(e.call));
+        else if (isEntry(e, JT.attention) && e.item.kind === "question")
+            open.add(`${e.item.id}\0${e.item.rev}`);
+        else if (isEntry(e, JT.attentionResolved))
+            open.delete(`${e.id}\0${e.rev}`);
+    }
+    if (!done || !["done", "failed", "stopped"].includes(String(done.status)) || open.size)
+        return false;
+    return [...calls].every(c => ended.has(c));
+}
+export class EventPump {
+    options;
+    log;
+    chain = Promise.resolve();
+    marks = new Map();
+    /** Sources whose watermark moved since the log last recorded it. */
+    unsaved = new Set();
+    dirty = new Set();
+    identities = new Map();
+    timer;
+    closed = false;
+    compactedAt = 0;
+    reported;
+    identityScan = -1;
+    /** Set by open() when the log was created (or replaced after corruption) and everything on disk was derived. */
+    backfilled = false;
+    /** Resolves once open() (or close()) ran: emits wait for it. */
+    ready;
+    isReady;
+    constructor(options) {
+        this.options = options;
+        this.ready = new Promise(resolve => { this.isReady = resolve; });
+        options.orch.onAppend = () => this.kick();
+        options.store.appended = wid => this.kick(wid);
+    }
+    /** The log's epoch and head (after open). */
+    get head() { return this.log && { epoch: this.log.epoch, head: this.log.head, dropped: this.log.dropped }; }
+    serial(fn) {
+        const run = this.chain.then(fn);
+        this.chain = run.catch(() => { });
+        return run;
+    }
+    /** Orchestrator start, after the store recovered its workflows: open (or create) the log, compact, then derive
+     *  everything after the watermarks (on a new log: everything still on disk). */
+    open() {
+        return this.serial(async () => {
+            if (this.closed || this.log)
+                return;
+            const opened = await EventLog.open(eventsLog(this.options.home));
+            if (opened.corrupt)
+                console.error(`durable-subagents: ${opened.corrupt}; a new event log (epoch ${opened.log.epoch}) starts`);
+            this.log = opened.log;
+            this.backfilled = opened.created;
+            // A source the store no longer has was pruned: its journal is gone and it is never derived again.
+            for (const [source, seq] of opened.marks)
+                if (source === "orch" || this.options.store.workflows.has(source))
+                    this.marks.set(source, seq);
+            await this.compact();
+            for (const wid of this.options.store.workflows.keys())
+                this.dirty.add(wid);
+            await this.pass().catch(() => { }); // logged; retried in the background
+        }).finally(() => this.isReady());
+    }
+    /** A source appended: derive soon (batched). */
+    kick(wid) {
+        if (wid !== undefined)
+            this.dirty.add(wid);
+        if (this.timer || this.closed)
+            return;
+        this.timer = setTimeout(() => { this.timer = undefined; this.serial(() => this.pass()).catch(() => { }); }, this.options.delayMs ?? 50);
+        this.timer.unref?.();
+    }
+    /** Derive and log everything appended so far (prune calls it before the journal goes). Rejects when that could not be
+     *  logged (or the log is not open): the caller must not drop the sources. */
+    flush() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = undefined;
+        }
+        return this.serial(async () => {
+            if (!this.log || this.closed)
+                throw new Error("event log not open");
+            await this.pass();
+        });
+    }
+    /** EventSink: append drafts (any type), echoing the workflow's request id and labels when the draft has none. */
+    async emit(drafts) {
+        await this.ready;
+        return this.serial(async () => {
+            // After close (the orchestrator exits) nothing is logged: R7 state is derived again by the next orchestrator.
+            if (!this.log || this.closed || !drafts.length)
+                return;
+            const log = await this.writer();
+            const filled = drafts.map(d => {
+                const id = this.identity(d.wid);
+                return { ...d, ...(d.request === undefined && id.request !== undefined ? { request: id.request } : {}), ...(d.labels === undefined && id.labels ? { labels: id.labels } : {}) };
+            });
+            await log.append(filled);
+        });
+    }
+    /** Orchestrator exit: derive what is left, record the watermarks, close the log. */
+    close() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = undefined;
+        }
+        const done = this.serial(async () => {
+            if (this.closed)
+                return;
+            try {
+                if (this.log) {
+                    await this.pass();
+                    if (this.unsaved.size) {
+                        await (await this.writer()).append([], this.saved());
+                        this.unsaved.clear();
+                    }
+                }
+            }
+            finally {
+                this.closed = true;
+                await this.log?.close();
+                if (this.timer) {
+                    clearTimeout(this.timer);
+                    this.timer = undefined;
+                }
+            }
+        }).finally(() => this.isReady());
+        return done.catch(error => console.error(`durable-subagents: event log close failed: ${String(error)}`));
+    }
+    saved(next) {
+        const out = new Map();
+        for (const source of this.unsaved) {
+            const seq = this.marks.get(source);
+            if (seq !== undefined)
+                out.set(source, seq);
+        }
+        for (const [source, seq] of next ?? [])
+            out.set(source, seq);
+        return out;
+    }
+    /** Request id and labels of a workflow's run request (cached while the workflow is not pruned). */
+    identity(wid) {
+        const cached = this.identities.get(wid);
+        if (cached)
+            return cached;
+        const orch = committed(this.options.orch);
+        // A miss rescans the ledger only after it grew (a journal may be derived before its `created` is appended).
+        if (orch.length === this.identityScan)
+            return {};
+        this.identityScan = orch.length;
+        // One scan resolves every workflow not cached yet (a backfill asks for all of them).
+        const wanted = new Map(), rids = new Map();
+        for (const e of orch)
+            if (e.type === JT.created && this.options.store.workflows.has(String(e.wid)) && !this.identities.has(String(e.wid))) {
+                wanted.set(String(e.wid), String(e.rid));
+                rids.set(String(e.rid), String(e.wid));
+            }
+        const found = new Map();
+        for (const [w, rid] of wanted)
+            found.set(w, requestId(rid) !== undefined ? { request: requestId(rid) } : {});
+        for (const e of orch) {
+            if (e.type !== "request")
+                continue;
+            const req = e.request, w = req && req.kind === "run" ? rids.get(req.rid) : undefined;
+            const labels = w !== undefined ? labelsOf(req.body) : undefined;
+            if (labels)
+                found.set(w, { ...found.get(w), labels });
+        }
+        // Only workflows with a `created` entry are cached: an earlier pass may see a journal before its ledger entry.
+        for (const [w, id] of found)
+            this.identities.set(w, id);
+        return this.identities.get(wid) ?? {};
+    }
+    /** The writer; a broken one (a failure after a compaction's rename) is reopened first: the open applies the start
+     *  skip. A log it had to create anew (corrupt or gone) is backfilled from everything on disk: the caller's drafts
+     *  were derived for the old one, so this throws (a pass retries from the cleared watermarks, R7 on its next tick). */
+    async writer() {
+        const log = this.log;
+        if (!log.broken)
+            return log;
+        await log.close();
+        const opened = await EventLog.open(eventsLog(this.options.home));
+        this.log = opened.log;
+        if (opened.corrupt)
+            console.error(`durable-subagents: ${opened.corrupt}; a new event log (epoch ${opened.log.epoch}) starts`);
+        if (opened.created) {
+            this.marks.clear();
+            this.unsaved.clear();
+            this.backfilled = true;
+            for (const wid of this.options.store.workflows.keys())
+                this.dirty.add(wid);
+            this.kick();
+            throw new Error(`event log replaced by a new epoch ${opened.log.epoch}: deriving everything again`);
+        }
+        return opened.log;
+    }
+    /** One derivation pass over the orchestrator ledger and the workflows appended to since the last one. Appends in
+     *  chunks of at most EVENT_SEQ_SKIP drafts, each with the watermarks of what it completes (a source cut inside a chunk
+     *  gets the position up to which its events are in it), so memory stays bounded and a failure re-derives only the
+     *  rest. Throws when a chunk could not be logged (after scheduling a retry). */
+    async pass() {
+        if (!this.log || this.closed)
+            return;
+        const dirty = [...this.dirty];
+        this.dirty.clear();
+        let drafts = [], forgotten = [];
+        const next = new Map();
+        // Log the chunk (with the watermarks it completes) and move the in-memory watermarks; a chunk without events only
+        // moves them (recorded with the next write, or at close).
+        const commit = async () => {
+            if (drafts.length) {
+                await (await this.writer()).append(drafts, this.saved(next));
+                this.unsaved.clear();
+            }
+            for (const [source, seq] of next) {
+                this.marks.set(source, seq);
+                if (!drafts.length)
+                    this.unsaved.add(source);
+            }
+            for (const wid of forgotten) {
+                this.marks.delete(wid);
+                this.identities.delete(wid);
+                this.unsaved.delete(wid);
+            }
+            drafts = [];
+            forgotten = [];
+            next.clear();
+        };
+        /** Add the drafts of source position `at`, committing first when they would overflow the chunk (`done` = the
+         *  watermark of that source before them). */
+        const add = async (source, done, derived) => {
+            if (!derived.length)
+                return;
+            if (drafts.length + derived.length > EVENT_SEQ_SKIP && drafts.length) {
+                if (done > (this.marks.get(source) ?? 0))
+                    next.set(source, done);
+                await commit();
+            }
+            drafts.push(...derived);
+        };
+        try {
+            const orch = committed(this.options.orch), from = this.marks.get("orch") ?? 0;
+            // The ledger first: a workflow's `submitted` precedes its journal's events.
+            for (let i = from; i < orch.length; i++) {
+                const e = orch[i];
+                if (e.type === JT.created && this.options.store.workflows.has(String(e.wid)))
+                    await add("orch", i, [deriveCreated(orch, i, this.identity(String(e.wid)))]);
+                else if (e.type === "pruned")
+                    forgotten.push(String(e.wid));
+            }
+            if (orch.length > from)
+                next.set("orch", orch.length);
+            for (const wid of dirty) {
+                const wf = this.options.store.workflows.get(wid);
+                if (!wf)
+                    continue;
+                const entries = committed(wf.journal), start = this.marks.get(wid) ?? 0;
+                if (entries.length <= start)
+                    continue;
+                const id = this.identity(wid);
+                for (let i = start; i < entries.length; i++)
+                    await add(wid, i, deriveEntry(wid, entries, i, orch, id));
+                next.set(wid, entries.length);
+            }
+            await commit();
+        }
+        catch (error) {
+            for (const wid of dirty)
+                this.dirty.add(wid);
+            const text = `durable-subagents: event log append failed (retrying): ${String(error)}`;
+            if (this.reported !== text) {
+                this.reported = text;
+                console.error(text);
+            }
+            if (!this.closed && !this.timer) {
+                this.timer = setTimeout(() => { this.timer = undefined; this.serial(() => this.pass()).catch(() => { }); }, 1000);
+                this.timer.unref?.();
+            }
+            throw error;
+        }
+        if (Date.now() - this.compactedAt >= COMPACT_EVERY_MS)
+            await this.compact();
+    }
+    /** Retention: drop events older than the window whose workflow is quiet (or pruned); at start and at most hourly. */
+    async compact() {
+        if (!this.log)
+            return;
+        this.compactedAt = Date.now();
+        const window = this.options.retentionMs?.() ?? EVENT_RETENTION_MS, cutoff = Date.now() - window;
+        const pruned = new Set(), orch = committed(this.options.orch);
+        for (const e of orch)
+            if (e.type === "pruned")
+                pruned.add(String(e.wid));
+        const verdicts = new Map();
+        const droppable = (wid) => {
+            let v = verdicts.get(wid);
+            if (v === undefined) {
+                const wf = this.options.store.workflows.get(wid);
+                v = wf ? quiet(wid, committed(wf.journal)) : pruned.has(wid);
+                verdicts.set(wid, v);
+            }
+            return v;
+        };
+        try {
+            // The watermarks written are the ones durable now plus those moved since (both are true of the derived log).
+            const marks = new Map(this.marks);
+            const dropped = await (await this.writer()).compact(r => r.at < cutoff && droppable(String(r.e.wid)), marks);
+            if (dropped)
+                this.unsaved.clear();
+        }
+        catch (error) {
+            console.error(`durable-subagents: event log compaction failed: ${String(error)}`);
+        }
+    }
+}