sealkeep 0.8.1 → 0.11.0

package/dist/src/watcher.d.ts

@@ -1,4 +1,4 @@
-import { type AgentId } from "./adapters.js";
+import { detectAgents, findTranscripts, type AgentId } from "./adapters.js";
 import { ArchiveQueue, type ArchiveJob } from "./queue.js";
 export type WatcherEvent = {
     agent: AgentId;
@@ -10,10 +10,16 @@ export type WatcherEvent = {
 export type WatcherOptions = {
     home?: string;
     agents?: AgentId[];
+    /** Overrides saved path exclusions, principally for isolated callers/tests. */
+    exclusions?: string[];
     /** Wait for a file to stop changing before queuing it. */
     debounceMs?: number;
     queue?: ArchiveQueue;
     onEvent?: (event: WatcherEvent) => void;
+    /** Dependency seam for deterministic close/reconciliation tests. */
+    detectAgents?: typeof detectAgents;
+    /** Dependency seam for deterministic scan lifecycle tests. */
+    findTranscripts?: typeof findTranscripts;
 };
 export type TranscriptWatcher = {
     roots: string[];

package/dist/src/watcher.js

@@ -1,8 +1,9 @@
 import { watch } from "node:fs";
 import { stat } from "node:fs/promises";
-import { join, resolve } from "node:path";
-import { detectAgents } from "./adapters.js";
+import { basename, join, resolve } from "node:path";
+import { detectAgents, findTranscripts } from "./adapters.js";
 import { ArchiveQueue } from "./queue.js";
+import { localSettingsPath, readLocalSettings, transcriptPathIsExcluded } from "./machine-settings.js";
 import { applySyncRules, resolveSyncRules } from "./sync-rules.js";
 /**
  * Watches adapter transcript roots and records archive intents.
@@ -14,20 +15,120 @@ import { applySyncRules, resolveSyncRules } from "./sync-rules.js";
 export async function startTranscriptWatcher(dataDir, options = {}) {
     const queue = options.queue ?? new ArchiveQueue(dataDir);
     const debounceMs = options.debounceMs ?? 2000;
-    const installs = (await detectAgents(options.home)).filter((install) => !options.agents || options.agents.includes(install.agent));
-    const watchers = [];
+    const settings = await readLocalSettings(dataDir);
+    let selectedAgents = new Set(options.agents ?? settings.watchedAgents);
+    const watchers = new Map();
     const timers = new Map();
+    // Returned by reference and mutated during reconciliation, so daemon status
+    // reflects a Settings change without replacing the TranscriptWatcher object.
     const roots = [];
     let closed = false;
+    let settingsWatcher;
+    let settingsTimer;
+    let reconciliation = Promise.resolve();
+    let closeInFlight;
     // Rules are resolved once at startup and refreshed only on scan — the daemon's
     // periodic reconciliation point — rather than per event, so a burst of writes
     // cannot turn into a burst of account lookups.
     let rules = (await resolveSyncRules(dataDir)).rules;
+    let exclusions = options.exclusions ?? settings.exclusions;
+    const excluded = (path) => transcriptPathIsExcluded(path, exclusions, options.home);
+    function schedule(agent, path) {
+        const existing = timers.get(path);
+        if (existing)
+            clearTimeout(existing.timer);
+        const timer = setTimeout(() => { timers.delete(path); void admitAndEnqueue(agent, path); }, debounceMs);
+        timer.unref?.();
+        timers.set(path, { agent, timer });
+    }
+    function attach(agent, root) {
+        if (closed)
+            return;
+        try {
+            const watcher = watch(root, { recursive: true }, (_event, filename) => {
+                if (!filename || !selectedAgents.has(agent))
+                    return;
+                const path = resolve(join(root, filename.toString()));
+                if (!path.endsWith(".jsonl") || excluded(path))
+                    return;
+                schedule(agent, path);
+            });
+            watcher.on("error", (error) => options.onEvent?.({ agent, path: root, error: error.message }));
+            watchers.set(root, { agent, watcher });
+        }
+        catch (error) {
+            options.onEvent?.({ agent, path: root, error: error instanceof Error ? error.message : "watch failed" });
+        }
+    }
+    /**
+     * Settings are live daemon policy, not merely restart configuration. Rebuild
+     * the small set of native roots at the same cadence as the safety scan: stop
+     * observing disabled agents, attach newly enabled/created stores, and cancel
+     * debounce work that was admitted under the previous selection.
+     */
+    async function reconcileRoots(refreshSettings) {
+        if (closed)
+            return;
+        if (refreshSettings) {
+            const latest = await readLocalSettings(dataDir);
+            if (options.agents === undefined)
+                selectedAgents = new Set(latest.watchedAgents);
+            if (options.exclusions === undefined)
+                exclusions = latest.exclusions;
+        }
+        const installs = await (options.detectAgents ?? detectAgents)(options.home);
+        const desired = new Map();
+        for (const install of installs) {
+            if (!selectedAgents.has(install.agent))
+                continue;
+            for (const root of install.transcriptRoots) {
+                const entry = await stat(root).catch(() => null);
+                if (entry?.isDirectory())
+                    desired.set(root, install.agent);
+            }
+        }
+        // close() may have run while settings, agent detection, or root stats were
+        // awaiting I/O. Mutating the native watcher map after that point would
+        // resurrect handles that no caller can close.
+        if (closed)
+            return;
+        for (const [root, current] of watchers) {
+            if (desired.get(root) === current.agent)
+                continue;
+            current.watcher.close();
+            watchers.delete(root);
+        }
+        for (const [root, agent] of desired)
+            if (!watchers.has(root))
+                attach(agent, root);
+        for (const [path, pending] of timers) {
+            if (selectedAgents.has(pending.agent))
+                continue;
+            clearTimeout(pending.timer);
+            timers.delete(path);
+        }
+        roots.splice(0, roots.length, ...[...watchers.keys()].sort());
+    }
+    function requestReconcile(refreshSettings) {
+        if (closed)
+            return Promise.resolve();
+        // A settings rename and a periodic scan can arrive together. Serialize
+        // their close/attach mutations so neither resurrects a root the other just
+        // disabled or loses a newly enabled one.
+        const run = reconciliation.then(() => reconcileRoots(refreshSettings));
+        reconciliation = run.catch(() => undefined);
+        return run;
+    }
     async function discover() {
-        const { findTranscripts } = await import("./adapters.js");
         const found = [];
-        for (const install of installs)
-            found.push(...await findTranscripts(install.agent, options.home, 500));
+        for (const agent of selectedAgents) {
+            if (closed)
+                break;
+            found.push(...await (options.findTranscripts ?? findTranscripts)(agent, options.home, undefined, {
+                exclude: excluded,
+                includeProjects: false,
+            }));
+        }
         return found;
     }
     /**
@@ -45,7 +146,7 @@ export async function startTranscriptWatcher(dataDir, options = {}) {
             return;
         try {
             const info = await stat(path).catch(() => null);
-            if (!info?.isFile())
+            if (closed || !info?.isFile())
                 return;
             const { job, deduped } = await queue.enqueue({ sourcePath: path, agent, event: "watcher" });
             options.onEvent?.({ agent, path, job, deduped });
@@ -58,64 +159,106 @@ export async function startTranscriptWatcher(dataDir, options = {}) {
     // live events, enqueueNow — so a session the sync rules exclude is never queued
     // no matter which of those triggered it.
     async function admitAndEnqueue(agent, path) {
-        if (closed || !(await admitted(path)))
+        if (closed || !selectedAgents.has(agent) || excluded(path) || !(await admitted(path)))
             return;
         await enqueue(agent, path);
     }
-    function schedule(agent, path) {
-        const existing = timers.get(path);
-        if (existing)
-            clearTimeout(existing);
-        const timer = setTimeout(() => { timers.delete(path); void admitAndEnqueue(agent, path); }, debounceMs);
-        timer.unref?.();
-        timers.set(path, timer);
-    }
-    for (const install of installs) {
-        for (const root of install.transcriptRoots) {
-            if (!(await stat(root).catch(() => null)))
-                continue;
-            roots.push({ agent: install.agent, root });
-            try {
-                const watcher = watch(root, { recursive: true }, (_event, filename) => {
-                    if (!filename)
-                        return;
-                    const path = resolve(join(root, filename.toString()));
-                    if (!path.endsWith(".jsonl"))
-                        return;
-                    schedule(install.agent, path);
-                });
-                watcher.on("error", (error) => options.onEvent?.({ agent: install.agent, path: root, error: error.message }));
-                watchers.push(watcher);
+    async function scanCurrent() {
+        if (closed)
+            return 0;
+        // Reconcile identity/policy first: a settings-triggered scan must detach a
+        // disabled store before it walks anything, and must attach an enabled store
+        // even if the account rules lookup is temporarily slow.
+        await requestReconcile(true);
+        if (closed)
+            return 0;
+        rules = (await resolveSyncRules(dataDir)).rules;
+        if (closed)
+            return 0;
+        const found = await discover();
+        if (closed)
+            return 0;
+        const included = new Set(applySyncRules(found, rules).included.map((item) => item.path));
+        const candidates = found.filter((candidate) => included.has(candidate.path));
+        const outcomes = await queue.enqueueMany(candidates.map((candidate) => ({
+            sourcePath: candidate.path, agent: candidate.agent, event: "watcher"
+        })));
+        for (const outcome of outcomes) {
+            const agent = outcome.request.agent;
+            if (outcome.ok) {
+                options.onEvent?.({ agent, path: outcome.request.sourcePath, job: outcome.job, deduped: outcome.deduped });
             }
-            catch (error) {
-                options.onEvent?.({ agent: install.agent, path: root, error: error instanceof Error ? error.message : "watch failed" });
+            else {
+                options.onEvent?.({ agent, path: outcome.request.sourcePath, error: outcome.error instanceof Error ? outcome.error.message : "enqueue failed" });
             }
         }
+        return found.length;
+    }
+    await requestReconcile(false);
+    // Settings are written by atomic rename, so watch the containing directory
+    // rather than the old inode. This lane is independent of the archive tick:
+    // disabling an agent closes its native watcher promptly even on a five-minute
+    // daemon schedule, and enabling one attaches as soon as its store exists.
+    if (options.agents === undefined || options.exclusions === undefined) {
+        const wanted = basename(localSettingsPath(dataDir));
+        try {
+            settingsWatcher = watch(dataDir, (_event, filename) => {
+                if (closed || (filename && basename(filename.toString()) !== wanted))
+                    return;
+                if (settingsTimer)
+                    clearTimeout(settingsTimer);
+                settingsTimer = setTimeout(() => {
+                    settingsTimer = undefined;
+                    // Enabling a store should also pick up transcripts already present;
+                    // otherwise the new root attaches promptly but its backlog still
+                    // waits for the daemon's long safety-scan interval.
+                    void scanCurrent().catch((error) => options.onEvent?.({
+                        agent: "codex",
+                        path: localSettingsPath(dataDir),
+                        error: error instanceof Error ? error.message : "settings reconciliation failed",
+                    }));
+                }, 25);
+                settingsTimer.unref?.();
+            });
+            settingsWatcher.on("error", (error) => options.onEvent?.({ agent: "codex", path: dataDir, error: error.message }));
+        }
+        catch (error) {
+            options.onEvent?.({ agent: "codex", path: dataDir, error: error instanceof Error ? error.message : "settings watch failed" });
+        }
     }
     return {
-        roots: roots.map((entry) => entry.root),
+        roots,
         enqueueNow: (agent, path) => admitAndEnqueue(agent, resolve(path)),
-        async scan() {
-            // The reconciliation point: pick up any change to the rules themselves,
-            // then decide admission once for the whole batch rather than once per
-            // candidate the way a live event has to.
-            rules = (await resolveSyncRules(dataDir)).rules;
-            const found = await discover();
-            const included = new Set(applySyncRules(found, rules).included.map((item) => item.path));
-            for (const candidate of found) {
-                if (included.has(candidate.path))
-                    await enqueue(candidate.agent, candidate.path);
-            }
-            return found.length;
-        },
+        scan: scanCurrent,
         pending: () => timers.size,
-        async close() {
+        close() {
+            if (closeInFlight)
+                return closeInFlight;
             closed = true;
-            for (const timer of timers.values())
-                clearTimeout(timer);
-            timers.clear();
-            for (const watcher of watchers)
-                watcher.close();
+            closeInFlight = (async () => {
+                const closeObservedHandles = () => {
+                    if (settingsTimer)
+                        clearTimeout(settingsTimer);
+                    settingsTimer = undefined;
+                    settingsWatcher?.close();
+                    settingsWatcher = undefined;
+                    for (const { timer } of timers.values())
+                        clearTimeout(timer);
+                    timers.clear();
+                    for (const { watcher } of watchers.values())
+                        watcher.close();
+                    watchers.clear();
+                    roots.splice(0, roots.length);
+                };
+                closeObservedHandles();
+                // A reconciliation which passed its first closed check before this
+                // call owns no handles after its final check, but join it before
+                // returning so close is a complete lifecycle boundary. The second
+                // sweep also protects older/injected reconciliation implementations.
+                await reconciliation;
+                closeObservedHandles();
+            })();
+            return closeInFlight;
         }
     };
 }

package/dist/src/worker.d.ts

@@ -1,11 +1,24 @@
 import { ArchiveQueue, type ArchiveJob } from "./queue.js";
+import { archiveFile } from "./vault.js";
 import { Reservations } from "./disk.js";
 /** Parallel seals are capped whatever the config asks: sealing is disk- and CPU-bound, and a machine short of space is rarely long on either. */
 export declare const MAX_SEAL_CONCURRENCY = 4;
+/** Long enough for a slow local chunk, short enough to expire before the
+ * ordinary five-minute queue lease can be mistaken for useful work. */
+export declare const DEFAULT_SEAL_IDLE_MS: number;
+export type ActiveSeal = {
+    project: string;
+    projectKey: string | null;
+    sourcePath: string;
+};
 export type DrainOptions = {
     queue?: ArchiveQueue;
     owner?: string;
     max?: number;
+    /** Off to seal without the end-of-drain active-project announcement (index sync + finished lines). */
+    announce?: boolean;
+    /** Daemon handoff: report active seals without starting another index writer. */
+    onActiveSealed?: (seals: readonly ActiveSeal[]) => void;
     /** Off only for tests that want to archive without a real filesystem behind them. */
     requireSpace?: boolean;
     /**
@@ -16,9 +29,8 @@ export type DrainOptions = {
      */
     reserveBytes?: number;
     /**
-     * How many seals may run at once. Defaults to a `sealConcurrency` field in
-     * the vault's config.json when one is present (readConfig tolerates and
-     * preserves fields it does not know), and to 1 otherwise. Clamped to
+     * Internal/test seam for how many seals may run at once. Production callers
+     * omit it and use the safe default of one. Clamped to
      * 1..MAX_SEAL_CONCURRENCY.
      */
     concurrency?: number;
@@ -35,6 +47,18 @@ export type DrainOptions = {
      * and off by default in archiveFile itself, so nothing else changes.
      */
     delta?: boolean;
+    /** Cancels the owned seal and leaves its queue row retryable. */
+    signal?: AbortSignal;
+    /** Maximum silence between source/provider byte progress events. */
+    sealIdleMs?: number;
+    /** Background-only duty cycle. Manual/foreground callers omit it. */
+    backgroundCpuTarget?: number;
+    /** Deterministic CPU scheduling seams. */
+    cpuUsage?: typeof process.cpuUsage;
+    now?: () => number;
+    sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
+    /** Deterministic liveness seam. Production always uses archiveFile. */
+    archive?: typeof archiveFile;
 };
 /**
  * Encrypts queued transcripts. The recovery phrase lives only in this process: