sealkeep 0.9.0 → 0.11.0

package/dist/src/context-background.d.ts

@@ -0,0 +1,37 @@
+import type { ContextRequestDrain } from "./agent-context.js";
+export type BackgroundContextDrain = {
+    pid: number | undefined;
+    /** Secret-free, never-paused orphan cleanup sibling. */
+    watchdogPid?: number;
+    started: Promise<void>;
+    result: Promise<ContextRequestDrain>;
+    /** Observe useful CPU or byte progress without exposing request contents. */
+    onProgress?(listener: () => void): () => void;
+    stop(): Promise<void>;
+    /** Immediately terminate this exact owned child; used only after its wall-clock deadline. */
+    forceStop(): Promise<void>;
+};
+export type ContextDrainStarter = (dataDir: string, phrase: string, options: {
+    limit: number;
+    /** Automatic recall is one low-priority CPU lane, not another foreground search. */
+    cpuTarget?: number;
+    /** Tests may lower this; production can never raise the reviewed ceiling. */
+    maxRssBytes?: number;
+    /** Deterministic process fixture; production always uses the package child. */
+    childPath?: string;
+    /** Deterministic failure fixture; production uses the package watchdog. */
+    watchdogPath?: string;
+}) => BackgroundContextDrain;
+/** Automatic context may read the same large derived index as a rebuild, so it
+ * gets the same hard process boundaries. The hook itself never enters this
+ * process; it only leaves a tiny encrypted request for the daemon. */
+export declare const BACKGROUND_CONTEXT_MAX_OLD_SPACE_MB = 1024;
+export declare const BACKGROUND_CONTEXT_MAX_SEMI_SPACE_MB = 16;
+export declare const BACKGROUND_CONTEXT_MAX_RSS_BYTES: number;
+export declare const BACKGROUND_CONTEXT_CPU_TARGET = 0.3;
+export declare function backgroundContextExecArgv(parent: readonly string[]): string[];
+/**
+ * Give hosted discovery, provider refresh and index parsing a killable process
+ * boundary. The recovery phrase uses the owned IPC pipe, never argv or env.
+ */
+export declare const startBackgroundContextDrain: ContextDrainStarter;

package/dist/src/context-background.js

@@ -0,0 +1,309 @@
+import { fork } from "node:child_process";
+import { constants, setPriority } from "node:os";
+import { fileURLToPath } from "node:url";
+import { BACKGROUND_CPU_TOTAL_TARGET, signalOwnedBackgroundProcess, onOwnedBackgroundProcessExit, startBackgroundProcessCpuGovernor } from "./background-cpu.js";
+import { backgroundIndexExecArgv, backgroundIndexWorkerEnvironment } from "./index-background.js";
+import { daemonProcessIdentity, registerDaemonContextWorker } from "./daemon-lease.js";
+import { backgroundWorkerErrorFromWire } from "./background-worker-error.js";
+/** Automatic context may read the same large derived index as a rebuild, so it
+ * gets the same hard process boundaries. The hook itself never enters this
+ * process; it only leaves a tiny encrypted request for the daemon. */
+export const BACKGROUND_CONTEXT_MAX_OLD_SPACE_MB = 1024;
+export const BACKGROUND_CONTEXT_MAX_SEMI_SPACE_MB = 16;
+export const BACKGROUND_CONTEXT_MAX_RSS_BYTES = 1536 * 1024 * 1024;
+export const BACKGROUND_CONTEXT_CPU_TARGET = 0.30;
+export function backgroundContextExecArgv(parent) {
+    // Match historical indexing's reviewed loader allowlist; arbitrary preloads,
+    // inspector options and inline programs are not background-work inputs.
+    const inherited = backgroundIndexExecArgv(parent).filter((arg) => !arg.startsWith("--max-old-space-size") && !arg.startsWith("--max-semi-space-size")
+        && arg !== "--single-threaded-gc");
+    return [
+        ...inherited,
+        `--max-old-space-size=${BACKGROUND_CONTEXT_MAX_OLD_SPACE_MB}`,
+        `--max-semi-space-size=${BACKGROUND_CONTEXT_MAX_SEMI_SPACE_MB}`,
+        // Context parsing can trigger V8 collector threads outside JavaScript's
+        // cooperative checkpoints. Serial GC lets the parent's wall-duty governor
+        // bound the whole owned process rather than only its JavaScript lane.
+        "--single-threaded-gc",
+    ];
+}
+/**
+ * Give hosted discovery, provider refresh and index parsing a killable process
+ * boundary. The recovery phrase uses the owned IPC pipe, never argv or env.
+ */
+export const startBackgroundContextDrain = (dataDir, phrase, options) => {
+    const extension = import.meta.url.endsWith(".ts") ? ".ts" : ".js";
+    const childPath = options.childPath ?? fileURLToPath(new URL(`./context-drain-child${extension}`, import.meta.url));
+    const child = fork(childPath, [], {
+        stdio: ["ignore", "ignore", "inherit", "ipc"],
+        execArgv: backgroundContextExecArgv(process.execArgv),
+        env: backgroundIndexWorkerEnvironment(),
+    });
+    // The context event loop cannot observe IPC disconnect while SIGSTOP-paused.
+    // Arm the same generation-checked, secret-free sibling used by indexing
+    // before sending any work or permitting the first pause.
+    const watchdogEnv = {};
+    for (const key of ["PATH", "SystemRoot", "WINDIR", "TEMP", "TMP", "TMPDIR"]) {
+        if (process.env[key] !== undefined)
+            watchdogEnv[key] = process.env[key];
+    }
+    let watchdog;
+    try {
+        watchdog = fork(options.watchdogPath ?? fileURLToPath(new URL(`./index-background-watchdog${extension}`, import.meta.url)), [], {
+            stdio: ["ignore", "ignore", "inherit", "ipc"],
+            env: watchdogEnv,
+            execArgv: [...backgroundContextExecArgv(process.execArgv).filter((arg) => !arg.startsWith("--max-old-space-size")), "--max-old-space-size=64"],
+        });
+    }
+    catch (error) {
+        child.once("error", () => { });
+        signalOwnedBackgroundProcess(child, "SIGKILL");
+        throw error;
+    }
+    let watchdogDisarmed = false;
+    let watchdogExited = false;
+    let watchdogFailure;
+    let watchdogExitTimer;
+    let watchdogClosedResolve;
+    const watchdogClosed = new Promise((resolve) => { watchdogClosedResolve = resolve; });
+    // Recall is useful background preparation, but Codex/Claude and the daemon's
+    // heartbeat are interactive. Let the OS prefer them where process priority
+    // is supported; inability to adjust priority is not a correctness failure.
+    if (child.pid !== undefined) {
+        try {
+            setPriority(child.pid, constants.priority.PRIORITY_BELOW_NORMAL);
+        }
+        catch { /* advisory on platforms/containers that do not permit renice */ }
+    }
+    const cpuTarget = Math.min(1, Math.max(0.05, options.cpuTarget ?? BACKGROUND_CONTEXT_CPU_TARGET));
+    // Cooperative checkpoints cannot interrupt stop-the-world/native V8 work.
+    // The parent owns this exact child, so on POSIX it also enforces the duty
+    // cycle with short SIGSTOP/SIGCONT windows. Windows keeps the cooperative
+    // controller because it has no equivalent process signals.
+    let cpuGovernor = { stop() { } };
+    let startedResolve;
+    const progressListeners = new Set();
+    const reportProgress = () => {
+        for (const listener of progressListeners)
+            listener();
+    };
+    let startedSettled = false;
+    const started = new Promise((resolve) => { startedResolve = resolve; });
+    const markStarted = () => {
+        if (startedSettled)
+            return;
+        startedSettled = true;
+        startedResolve();
+    };
+    let resultResolve;
+    let resultReject;
+    let resultSettled = false;
+    const result = new Promise((resolve, reject) => {
+        resultResolve = resolve;
+        resultReject = reject;
+    });
+    let reportedResult;
+    let reportedError;
+    let governorFailure;
+    let workerIdentity;
+    let workerRegistration;
+    let closedResolve;
+    let closedSettled = false;
+    const closed = new Promise((resolve) => { closedResolve = resolve; });
+    let cleanedResolve;
+    const cleaned = new Promise((resolve) => { cleanedResolve = resolve; });
+    onOwnedBackgroundProcessExit(child, (code, signal) => {
+        cpuGovernor.stop();
+        clearTimeout(startupTimer);
+        closedSettled = true;
+        watchdogDisarmed = true;
+        if (!watchdogExited) {
+            if (watchdog.connected)
+                watchdog.send({ type: "disarm" }, () => { });
+            watchdogExitTimer = setTimeout(() => signalOwnedBackgroundProcess(watchdog, "SIGKILL"), 2_000);
+        }
+        closedResolve();
+        markStarted();
+        // Successful daemon passes do not call stop(); keep the pass in flight
+        // until both owned processes are gone, not merely until the worker exits.
+        void watchdogClosed.then(async () => {
+            try {
+                await (await workerRegistration)?.retireAfterExit();
+            }
+            catch (error) {
+                reportedError = error instanceof Error ? error : new Error(String(error));
+                reportedResult = undefined;
+            }
+            cleanedResolve();
+            if (resultSettled)
+                return;
+            resultSettled = true;
+            if (watchdogFailure)
+                resultReject(watchdogFailure);
+            else if (governorFailure)
+                resultReject(governorFailure);
+            else if (reportedResult)
+                resultResolve(reportedResult);
+            else
+                resultReject(reportedError ?? new Error(`Context child exited before reporting a result (${signal ?? code ?? "unknown"})`));
+        });
+    });
+    child.once("error", (error) => {
+        reportedError ??= error;
+        cpuGovernor.stop();
+    });
+    child.on("message", (wire) => {
+        if (!wire || typeof wire !== "object" || !("type" in wire))
+            return;
+        const message = wire;
+        if (message.type === "started" || message.type === "progress") {
+            markStarted();
+            reportProgress();
+            return;
+        }
+        if (message.type === "result") {
+            reportedResult = message.result;
+            return;
+        }
+        if (message.type === "error") {
+            reportedError = backgroundWorkerErrorFromWire(message.error);
+        }
+    });
+    const beginDrain = () => {
+        if (closedSettled || stopInFlight || watchdogFailure)
+            return;
+        clearTimeout(startupTimer);
+        // Recall is latency work, not bulk work: an agent's next tool boundary is
+        // waiting on it and the whole job is a second or two of CPU. It therefore
+        // runs at full share on its own, outside the daemon-wide background budget.
+        // Sharing that budget with the index catch-up halved its slice, the
+        // governor held it SIGSTOPped most of every second, a stopped child reports
+        // no progress, and the stall deadline killed every pass — for as long as the
+        // catch-up ran, which on a real vault is hours. No memory ever arrived.
+        cpuGovernor = startBackgroundProcessCpuGovernor({
+            target: 1, signal: (signal) => signalOwnedBackgroundProcess(child, signal),
+            verifySignals: true,
+            onFailure: (error) => {
+                governorFailure ??= error;
+                reportedResult = undefined;
+                // IPC loss is a separate cancellation channel when OS policy denies
+                // signals. The native child aborts and exits within its orphan grace.
+                if (child.connected)
+                    child.disconnect();
+                void stop();
+            },
+        });
+        if (governorFailure)
+            return;
+        child.send({
+            type: "drain",
+            dataDir,
+            phrase,
+            limit: options.limit,
+            // Avoid compounding two clocks on POSIX; the parent governor covers native
+            // work too. Windows falls back to the cooperative in-child controller.
+            cpuTarget: process.platform === "win32" ? Math.min(cpuTarget, BACKGROUND_CPU_TOTAL_TARGET / 3) : 1,
+            maxRssBytes: Math.min(BACKGROUND_CONTEXT_MAX_RSS_BYTES, Math.max(1, options.maxRssBytes ?? BACKGROUND_CONTEXT_MAX_RSS_BYTES)),
+        }, (error) => {
+            if (!error)
+                return;
+            reportedError ??= error;
+            void stop();
+        });
+    };
+    let stopInFlight;
+    const stop = () => {
+        if (stopInFlight)
+            return stopInFlight;
+        stopInFlight = (async () => {
+            clearTimeout(startupTimer);
+            if (closedSettled) {
+                await cleaned;
+                return;
+            }
+            cpuGovernor.stop();
+            signalOwnedBackgroundProcess(child, "SIGTERM");
+            let force;
+            await Promise.race([
+                closed,
+                new Promise((resolve) => {
+                    force = setTimeout(() => {
+                        if (!closedSettled) {
+                            cpuGovernor.stop();
+                            signalOwnedBackgroundProcess(child, "SIGKILL");
+                        }
+                        resolve();
+                    }, 2_000);
+                }),
+            ]);
+            if (force)
+                clearTimeout(force);
+            if (!closedSettled)
+                await closed;
+            await cleaned;
+        })();
+        return stopInFlight;
+    };
+    const forceStop = () => {
+        clearTimeout(startupTimer);
+        cpuGovernor.stop();
+        if (!closedSettled)
+            signalOwnedBackgroundProcess(child, "SIGKILL");
+        if (!stopInFlight)
+            stopInFlight = cleaned;
+        return stopInFlight;
+    };
+    const watchdogFailed = (error) => {
+        if (closedSettled || watchdogDisarmed)
+            return;
+        watchdogFailure ??= error;
+        reportedResult = undefined;
+        void stop();
+    };
+    watchdog.once("error", watchdogFailed);
+    watchdog.once("disconnect", () => watchdogFailed(new Error("Automatic context orphan watchdog disconnected unexpectedly")));
+    onOwnedBackgroundProcessExit(watchdog, (code, signal) => {
+        watchdogExited = true;
+        if (watchdogExitTimer)
+            clearTimeout(watchdogExitTimer);
+        watchdogClosedResolve();
+        watchdogFailed(new Error(`Automatic context orphan watchdog exited unexpectedly (${signal ?? code ?? "unknown"})`));
+    });
+    watchdog.once("message", (wire) => {
+        if (wire && typeof wire === "object" && "type" in wire && wire.type === "armed") {
+            if (closedSettled || stopInFlight || !workerIdentity || child.pid === undefined)
+                return;
+            workerRegistration = registerDaemonContextWorker(dataDir, child.pid, workerIdentity);
+            void workerRegistration.then(beginDrain, watchdogFailed);
+        }
+        else
+            watchdogFailed(new Error("Automatic context orphan watchdog did not acknowledge ownership"));
+    });
+    const startupTimer = setTimeout(() => watchdogFailed(new Error("Automatic context orphan watchdog startup timed out")), 10_000);
+    void (async () => {
+        if (child.pid === undefined)
+            throw new Error("Automatic context child has no process identity");
+        const identity = await daemonProcessIdentity(child.pid);
+        if (closedSettled || stopInFlight)
+            return;
+        if (identity.state !== "identified")
+            throw new Error("Automatic context child process generation could not be verified");
+        workerIdentity = identity.identity;
+        watchdog.send({ type: "arm", pid: child.pid, identity: identity.identity, graceMs: 2_000 }, (error) => {
+            if (error)
+                watchdogFailed(error);
+        });
+    })().catch(watchdogFailed);
+    return {
+        pid: child.pid,
+        watchdogPid: watchdog.pid,
+        started,
+        result,
+        onProgress(listener) {
+            progressListeners.add(listener);
+            return () => { progressListeners.delete(listener); };
+        },
+        stop,
+        forceStop,
+    };
+};

package/dist/src/context-drain-child.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/context-drain-child.js

@@ -0,0 +1,98 @@
+import { drainAutomaticAgentContextRequests } from "./agent-context.js";
+import { createBackgroundCpuYield } from "./background-cpu.js";
+import { BACKGROUND_CONTEXT_CPU_TARGET, BACKGROUND_CONTEXT_MAX_RSS_BYTES, } from "./context-background.js";
+import { SealkeepError } from "./errors.js";
+import { backgroundUploadBytesPerSecond, readLocalSettings } from "./machine-settings.js";
+import { backgroundWorkerErrorToWire } from "./background-worker-error.js";
+function send(message) {
+    return new Promise((resolve) => {
+        if (!process.send || !process.connected) {
+            resolve();
+            return;
+        }
+        process.send(message, () => resolve());
+    });
+}
+let completed = false;
+const cancellation = new AbortController();
+const cancel = () => {
+    if (completed || cancellation.signal.aborted)
+        return;
+    const error = new Error("Automatic context owner requested shutdown");
+    error.name = "AbortError";
+    cancellation.abort(error);
+};
+process.once("SIGTERM", cancel);
+process.once("SIGINT", cancel);
+process.once("disconnect", () => {
+    if (completed)
+        return;
+    cancel();
+    setTimeout(() => process.exit(0), 2_000).unref();
+});
+// Recall can touch local indexes and optional hosted providers. Keeping one
+// request in one owned process gives the daemon a real cancellation boundary:
+// a fetch or parser that never settles cannot pin every later memory request.
+process.once("message", async (wire) => {
+    const request = wire && typeof wire === "object" ? wire : {};
+    if (request.type !== "drain" || typeof request.dataDir !== "string"
+        || typeof request.phrase !== "string" || typeof request.limit !== "number") {
+        await send({ type: "error", error: { message: "Context child received an invalid drain request" } });
+        completed = true;
+        if (process.connected)
+            process.disconnect();
+        return;
+    }
+    await send({ type: "started" });
+    try {
+        const settings = await readLocalSettings(request.dataDir);
+        const maxBytesPerSecond = backgroundUploadBytesPerSecond(settings);
+        const cpuTarget = typeof request.cpuTarget === "number" && Number.isFinite(request.cpuTarget)
+            ? Math.min(1, Math.max(0.05, request.cpuTarget))
+            : BACKGROUND_CONTEXT_CPU_TARGET;
+        const maxRssBytes = typeof request.maxRssBytes === "number" && Number.isFinite(request.maxRssBytes)
+            ? Math.min(BACKGROUND_CONTEXT_MAX_RSS_BYTES, Math.max(1, request.maxRssBytes))
+            : BACKGROUND_CONTEXT_MAX_RSS_BYTES;
+        const cooperate = createBackgroundCpuYield({ target: cpuTarget, signal: cancellation.signal, budget: null });
+        let lastProgressAt = 0;
+        const reportProgress = () => {
+            const now = Date.now();
+            if (now - lastProgressAt < 250)
+                return;
+            lastProgressAt = now;
+            void send({ type: "progress" });
+        };
+        const onYield = async () => {
+            cancellation.signal.throwIfAborted();
+            const before = process.memoryUsage().rss;
+            if (before > maxRssBytes) {
+                throw new SealkeepError("operation_timeout", `Automatic context stopped at its ${Math.round(maxRssBytes / 1024 / 1024)} MiB memory safety ceiling`, { operation: "automatic_context", rssBytes: before, maxRssBytes, retryable: true });
+            }
+            await cooperate();
+            const after = process.memoryUsage().rss;
+            if (after > maxRssBytes) {
+                throw new SealkeepError("operation_timeout", `Automatic context stopped at its ${Math.round(maxRssBytes / 1024 / 1024)} MiB memory safety ceiling`, { operation: "automatic_context", rssBytes: after, maxRssBytes, retryable: true });
+            }
+            reportProgress();
+        };
+        await onYield();
+        const result = await drainAutomaticAgentContextRequests(request.dataDir, request.phrase, {
+            limit: Math.max(1, Math.min(8, Math.trunc(request.limit))),
+            scheduling: {
+                signal: cancellation.signal,
+                maxBytesPerSecond,
+                onYield,
+                onProgress: reportProgress,
+            },
+        });
+        await send({ type: "result", result });
+    }
+    catch (cause) {
+        await send({ type: "error", error: backgroundWorkerErrorToWire(cause) });
+    }
+    finally {
+        completed = true;
+        if (process.connected)
+            process.disconnect();
+    }
+});

package/dist/src/context-reader.d.ts

@@ -0,0 +1,118 @@
+import { type SourceReference, type SourceReadLimits } from "./source-reader.js";
+export type ContextMessage = {
+    messageId: string;
+    role: "user" | "assistant" | "tool" | "unknown";
+    kind: "message" | "tool_call" | "tool_result";
+    /** The row is in this source; recalled text still belongs to another memory. */
+    provenance: "session" | "retrieved_memory" | "unknown";
+    toolName?: string;
+    text: string;
+    byteStart: number;
+    byteEnd: number;
+    turnId?: string;
+    callId?: string;
+    parentCallId?: string;
+    actualUserMessage?: true;
+    userEvidence?: "native_user_event" | "native_user_text" | "claude_user_message";
+    userContentIndexes?: number[];
+    textOffset?: number;
+    omittedPrefix?: true;
+    truncated?: boolean;
+};
+export type ContextCursor = {
+    sourceSha256: string;
+    byteStart: number;
+    textOffset?: number;
+};
+export type ReadContextOptions = SourceReadLimits & {
+    source: SourceReference;
+    query?: string;
+    byteStart?: number;
+    chunkIndex?: number;
+    cursor?: ContextCursor;
+    maxTokens?: number;
+    neighboringMessages?: number;
+};
+/** Byte locations refer to the immutable original JSONL, never reconstructed prose. */
+export declare function parseContextMessages(bytes: Buffer, source: SourceReference, byteStart?: number): ContextMessage[];
+/** Conservative byte accounting bounds text tokens without depending on a model tokenizer. */
+export declare function readContext(dataDir: string, phrase: string, options: ReadContextOptions): Promise<{
+    status: "unsupported";
+    source: SourceReference;
+    messages: never[];
+    reason: {};
+    message: string;
+    continuation?: undefined;
+    truncated?: undefined;
+} | {
+    status: "unsupported";
+    source: SourceReference;
+    messages: never[];
+    reason: string;
+    message: string;
+    continuation: {
+        sourceSha256: string;
+        byteStart: number;
+    };
+    truncated: boolean;
+} | {
+    continuation: ContextCursor;
+    truncated: boolean;
+    messages: ContextMessage[];
+    status: "ok";
+    source: SourceReference;
+    extractionVersion: number;
+    userAnchor: {
+        status: "observed";
+        messageId: string;
+        byteStart: number;
+        reason?: undefined;
+    } | {
+        status: "unavailable";
+        reason: string;
+        messageId?: undefined;
+        byteStart?: undefined;
+    };
+    range: {
+        byteStart: number;
+        byteEnd: number;
+    };
+    usage: {
+        wireBytes: number;
+        decodedBytes: number;
+        tokenBudget: number;
+        accounting: string;
+    };
+    reason?: undefined;
+    message?: undefined;
+} | {
+    truncated: boolean;
+    messages: ContextMessage[];
+    status: "ok";
+    source: SourceReference;
+    extractionVersion: number;
+    userAnchor: {
+        status: "observed";
+        messageId: string;
+        byteStart: number;
+        reason?: undefined;
+    } | {
+        status: "unavailable";
+        reason: string;
+        messageId?: undefined;
+        byteStart?: undefined;
+    };
+    range: {
+        byteStart: number;
+        byteEnd: number;
+    };
+    usage: {
+        wireBytes: number;
+        decodedBytes: number;
+        tokenBudget: number;
+        accounting: string;
+    };
+    reason?: undefined;
+    message?: undefined;
+    continuation?: undefined;
+}>;