sealkeep 0.5.0

package/dist/src/watcher.d.ts

@@ -0,0 +1,34 @@
+import { type AgentId } from "./adapters.js";
+import { ArchiveQueue, type ArchiveJob } from "./queue.js";
+export type WatcherEvent = {
+    agent: AgentId;
+    path: string;
+    job?: ArchiveJob;
+    deduped?: boolean;
+    error?: string;
+};
+export type WatcherOptions = {
+    home?: string;
+    agents?: AgentId[];
+    /** Wait for a file to stop changing before queuing it. */
+    debounceMs?: number;
+    queue?: ArchiveQueue;
+    onEvent?: (event: WatcherEvent) => void;
+};
+export type TranscriptWatcher = {
+    roots: string[];
+    /** Queues a path immediately, bypassing the debounce. Used by tests and manual scans. */
+    enqueueNow(agent: AgentId, path: string): Promise<void>;
+    /** Scans the watched roots once, so a restart catches files that changed while it was down. */
+    scan(): Promise<number>;
+    pending(): number;
+    close(): Promise<void>;
+};
+/**
+ * Watches adapter transcript roots and records archive intents.
+ *
+ * The watcher only ever enqueues; it never reads transcript contents and never
+ * encrypts. Because job ids are content-addressed, a burst of write events for one
+ * session collapses into a single job even before the debounce is considered.
+ */
+export declare function startTranscriptWatcher(dataDir: string, options?: WatcherOptions): Promise<TranscriptWatcher>;

package/dist/src/watcher.js

@@ -0,0 +1,121 @@
+import { watch } from "node:fs";
+import { stat } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { detectAgents } from "./adapters.js";
+import { ArchiveQueue } from "./queue.js";
+import { applySyncRules, resolveSyncRules } from "./sync-rules.js";
+/**
+ * Watches adapter transcript roots and records archive intents.
+ *
+ * The watcher only ever enqueues; it never reads transcript contents and never
+ * encrypts. Because job ids are content-addressed, a burst of write events for one
+ * session collapses into a single job even before the debounce is considered.
+ */
+export async function startTranscriptWatcher(dataDir, options = {}) {
+    const queue = options.queue ?? new ArchiveQueue(dataDir);
+    const debounceMs = options.debounceMs ?? 2000;
+    const installs = (await detectAgents(options.home)).filter((install) => !options.agents || options.agents.includes(install.agent));
+    const watchers = [];
+    const timers = new Map();
+    const roots = [];
+    let closed = false;
+    // Rules are resolved once at startup and refreshed only on scan — the daemon's
+    // periodic reconciliation point — rather than per event, so a burst of writes
+    // cannot turn into a burst of account lookups.
+    let rules = (await resolveSyncRules(dataDir)).rules;
+    async function discover() {
+        const { findTranscripts } = await import("./adapters.js");
+        const found = [];
+        for (const install of installs)
+            found.push(...await findTranscripts(install.agent, options.home, 500));
+        return found;
+    }
+    /**
+     * A live event knows only the one path that changed, so under "recent" its
+     * admission has to be re-derived against its current siblings; "all" and
+     * "settled" need no such lookup.
+     */
+    async function admitted(path) {
+        if (rules.mode === "all")
+            return true;
+        return applySyncRules(await discover(), rules).included.some((item) => item.path === path);
+    }
+    async function enqueue(agent, path) {
+        if (closed)
+            return;
+        try {
+            const info = await stat(path).catch(() => null);
+            if (!info?.isFile())
+                return;
+            const { job, deduped } = await queue.enqueue({ sourcePath: path, agent, event: "watcher" });
+            options.onEvent?.({ agent, path, job, deduped });
+        }
+        catch (error) {
+            options.onEvent?.({ agent, path, error: error instanceof Error ? error.message : "enqueue failed" });
+        }
+    }
+    // The one choke point every path onto the queue runs through outside a scan —
+    // live events, enqueueNow — so a session the sync rules exclude is never queued
+    // no matter which of those triggered it.
+    async function admitAndEnqueue(agent, path) {
+        if (closed || !(await admitted(path)))
+            return;
+        await enqueue(agent, path);
+    }
+    function schedule(agent, path) {
+        const existing = timers.get(path);
+        if (existing)
+            clearTimeout(existing);
+        const timer = setTimeout(() => { timers.delete(path); void admitAndEnqueue(agent, path); }, debounceMs);
+        timer.unref?.();
+        timers.set(path, timer);
+    }
+    for (const install of installs) {
+        for (const root of install.transcriptRoots) {
+            if (!(await stat(root).catch(() => null)))
+                continue;
+            roots.push({ agent: install.agent, root });
+            try {
+                const watcher = watch(root, { recursive: true }, (_event, filename) => {
+                    if (!filename)
+                        return;
+                    const path = resolve(join(root, filename.toString()));
+                    if (!path.endsWith(".jsonl"))
+                        return;
+                    schedule(install.agent, path);
+                });
+                watcher.on("error", (error) => options.onEvent?.({ agent: install.agent, path: root, error: error.message }));
+                watchers.push(watcher);
+            }
+            catch (error) {
+                options.onEvent?.({ agent: install.agent, path: root, error: error instanceof Error ? error.message : "watch failed" });
+            }
+        }
+    }
+    return {
+        roots: roots.map((entry) => entry.root),
+        enqueueNow: (agent, path) => admitAndEnqueue(agent, resolve(path)),
+        async scan() {
+            // The reconciliation point: pick up any change to the rules themselves,
+            // then decide admission once for the whole batch rather than once per
+            // candidate the way a live event has to.
+            rules = (await resolveSyncRules(dataDir)).rules;
+            const found = await discover();
+            const included = new Set(applySyncRules(found, rules).included.map((item) => item.path));
+            for (const candidate of found) {
+                if (included.has(candidate.path))
+                    await enqueue(candidate.agent, candidate.path);
+            }
+            return found.length;
+        },
+        pending: () => timers.size,
+        async close() {
+            closed = true;
+            for (const timer of timers.values())
+                clearTimeout(timer);
+            timers.clear();
+            for (const watcher of watchers)
+                watcher.close();
+        }
+    };
+}

package/dist/src/worker.d.ts

@@ -0,0 +1,52 @@
+import { ArchiveQueue, type ArchiveJob } from "./queue.js";
+import { Reservations } from "./disk.js";
+/** Parallel seals are capped whatever the config asks: sealing is disk- and CPU-bound, and a machine short of space is rarely long on either. */
+export declare const MAX_SEAL_CONCURRENCY = 4;
+export type DrainOptions = {
+    queue?: ArchiveQueue;
+    owner?: string;
+    max?: number;
+    /** Off only for tests that want to archive without a real filesystem behind them. */
+    requireSpace?: boolean;
+    /**
+     * How much room to leave alone. Overridable because a fixed 2 GB is right for
+     * a laptop and wrong for a small VM — and because a test that archives forty
+     * bytes should not fail on a machine that happens to be nearly full, which is
+     * exactly the machine this product is for.
+     */
+    reserveBytes?: number;
+    /**
+     * How many seals may run at once. Defaults to a `sealConcurrency` field in
+     * the vault's config.json when one is present (readConfig tolerates and
+     * preserves fields it does not know), and to 1 otherwise. Clamped to
+     * 1..MAX_SEAL_CONCURRENCY.
+     */
+    concurrency?: number;
+    /**
+     * The ledger concurrent seals coordinate their disk claims through.
+     * Injectable for tests; every real caller shares the per-process ledger so
+     * two drains in one process cannot promise each other's headroom away.
+     */
+    reservations?: Reservations;
+    /**
+     * Seal only what an appended transcript added since its newest proven
+     * archive (see archiveFile). On by default here — the worker re-archives the
+     * same growing sessions all day, which is exactly where re-sealing hurts —
+     * and off by default in archiveFile itself, so nothing else changes.
+     */
+    delta?: boolean;
+};
+/**
+ * Encrypts queued transcripts. The recovery phrase lives only in this process:
+ * it is never written to a job, a log line, or the local API.
+ *
+ * A job whose lease expired mid-archive is left for the next claim. Because
+ * archiving is idempotent by source bytes, the retry reuses the existing archive.
+ *
+ * With `concurrency` above 1, that many workers pull from the same claim loop.
+ * Each one reserves its disk claim *before* asking whether there is room, so
+ * two seals starting in the same moment cannot both be promised the same
+ * headroom — the race that filled a live disk to 0 bytes when a seal and a
+ * test suite each passed the per-job check against the same free space.
+ */
+export declare function drainQueue(dataDir: string, phrase: string, options?: DrainOptions): Promise<ArchiveJob[]>;

package/dist/src/worker.js

@@ -0,0 +1,190 @@
+import { createReadStream } from "node:fs";
+import { stat } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { hostname } from "node:os";
+import { resolve } from "node:path";
+import { ArchiveQueue } from "./queue.js";
+import { archiveFile, listArchives, readConfig } from "./vault.js";
+import { isVaultlineError } from "./errors.js";
+import { roomToArchive, sealReservations, spaceNeededFor } from "./disk.js";
+import { isV2 } from "./types.js";
+/** Parallel seals are capped whatever the config asks: sealing is disk- and CPU-bound, and a machine short of space is rarely long on either. */
+export const MAX_SEAL_CONCURRENCY = 4;
+/** Operational summary only: codes and messages, never transcript content. */
+function summarize(error) {
+    if (isVaultlineError(error))
+        return { code: error.code, message: error.message };
+    return { code: "internal", message: error instanceof Error ? error.message : "Unexpected error" };
+}
+const sha256OfFile = (path) => new Promise((done, failed) => {
+    const hash = createHash("sha256");
+    createReadStream(path).on("data", (piece) => hash.update(piece)).on("end", () => done(hash.digest("hex"))).on("error", failed);
+});
+/**
+ * The pressure valve: a transcript the disk cannot hold a sealed copy of
+ * streams STRAIGHT into a chunk folder instead of waiting for a reclaim —
+ * peak local cost is one chunk, and the archive is born offloaded and
+ * verified. This runs ONLY when the space guard has already refused the
+ * local seal, so the ordinary path (local seal, delta chains, offline
+ * archiving) is untouched; under pressure, full seals beat no seals.
+ */
+async function streamUnderPressure(dataDir, job, phrase) {
+    const config = await readConfig(dataDir);
+    if ((config.storageMode ?? "sealed") === "plain")
+        return { reason: "plain vaults keep readable archives on this disk" };
+    if (config.remoteStorage?.provider === "gdrive")
+        return { reason: "Google Drive stores archives as single objects, which need local room to build" };
+    // No remoteStorage entry is NOT "no storage": an enrolled managed vault
+    // carries a cloud token instead. sealToChunkFolder resolves the target the
+    // same way the rest of the product does, and its refusal becomes the reason.
+    // The dedup archiveFile would have done: an unchanged source that is
+    // already archived completes without sealing — or streaming — anything.
+    const absolute = resolve(job.sourcePath);
+    const size = await stat(absolute).then((entry) => entry.size).catch(() => -1);
+    const records = await listArchives(dataDir).catch(() => []);
+    const candidates = records.filter(isV2).filter((record) => record.source.path === absolute && record.source.bytes === size);
+    if (candidates.length > 0) {
+        const digest = await sha256OfFile(absolute).catch(() => null);
+        const match = digest ? candidates.find((record) => record.source.sha256 === digest) : undefined;
+        if (match)
+            return { record: match, deduplicated: true };
+    }
+    try {
+        const { claudeProjectFromPath, codexProjectFromRollout } = await import("./adapters.js");
+        const project = job.agent === "claude"
+            ? claudeProjectFromPath(absolute).project
+            : job.agent === "codex"
+                ? (await codexProjectFromRollout(absolute, new Date().toISOString())).project
+                : null;
+        const { sealToChunkFolder } = await import("./chunk-store.js");
+        const outcome = await sealToChunkFolder(dataDir, absolute, phrase, job.agent, { project });
+        return { record: outcome.record, deduplicated: false };
+    }
+    catch (error) {
+        return { reason: summarize(error).message };
+    }
+}
+/**
+ * A `sealConcurrency` in the vault's config.json, if someone put one there.
+ * The field is not part of the typed config yet; readConfig parses and keeps
+ * unknown fields, so a hand-added value is visible here. Anything unreadable
+ * or non-numeric means the default.
+ */
+async function configuredConcurrency(dataDir) {
+    try {
+        const config = (await readConfig(dataDir));
+        return typeof config.sealConcurrency === "number" ? config.sealConcurrency : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Encrypts queued transcripts. The recovery phrase lives only in this process:
+ * it is never written to a job, a log line, or the local API.
+ *
+ * A job whose lease expired mid-archive is left for the next claim. Because
+ * archiving is idempotent by source bytes, the retry reuses the existing archive.
+ *
+ * With `concurrency` above 1, that many workers pull from the same claim loop.
+ * Each one reserves its disk claim *before* asking whether there is room, so
+ * two seals starting in the same moment cannot both be promised the same
+ * headroom — the race that filled a live disk to 0 bytes when a seal and a
+ * test suite each passed the per-job check against the same free space.
+ */
+export async function drainQueue(dataDir, phrase, options = {}) {
+    const queue = options.queue ?? new ArchiveQueue(dataDir);
+    const owner = options.owner ?? `${hostname()}:${process.pid}`;
+    const max = options.max ?? 25;
+    const reservations = options.reservations ?? sealReservations;
+    const requested = options.concurrency ?? (await configuredConcurrency(dataDir)) ?? 1;
+    const concurrency = Math.min(MAX_SEAL_CONCURRENCY, Math.max(1, Math.trunc(requested) || 1));
+    const processed = [];
+    const refusedIds = new Set();
+    let budget = max;
+    async function work(workerOwner) {
+        while (budget > 0) {
+            budget -= 1;
+            const job = await queue.claim(workerOwner, refusedIds);
+            if (!job?.lease)
+                return;
+            const leaseId = job.lease.id;
+            // Do not start an archive there is no room to finish. Sealing writes the
+            // sealed object while the original is still on disk, and the machines
+            // this runs on are, by definition, the ones short of space. The claim is
+            // written into the ledger before the question is asked, so a sibling
+            // seal answering the same question in the same moment counts this job's
+            // bytes as already spoken for.
+            const guarded = options.requireSpace !== false;
+            if (guarded)
+                reservations.reserve(job.id, spaceNeededFor(job.sourceBytes, 0));
+            try {
+                if (guarded) {
+                    const room = await roomToArchive(dataDir, job.sourceBytes, options.reserveBytes, reservations, job.id);
+                    if (!room.ok) {
+                        // Before leaving it queued, try the pressure valve: stream the
+                        // seal straight into a chunk folder, whose peak local cost is one
+                        // chunk. The huge reservation is released first — the stream
+                        // never spends what the guard refused, and holding a claim this
+                        // size would starve every sibling seal in the pass.
+                        if (guarded)
+                            reservations.release(job.id);
+                        const streamed = await streamUnderPressure(dataDir, job, phrase);
+                        if ("record" in streamed) {
+                            processed.push(await queue.complete(job.id, leaseId, {
+                                archiveId: streamed.record.id, ciphertextSha256: streamed.record.cipher.ciphertextSha256,
+                                bytes: streamed.record.source.bytes, deduplicated: streamed.deduplicated
+                            }));
+                            continue;
+                        }
+                        // Left queued rather than failed: nothing is wrong with the
+                        // transcript, and it will archive on a later pass once something
+                        // has been reclaimed. Released, remembered, and stepped past —
+                        // an earlier version stopped the whole loop here, and one 1.7 GB
+                        // transcript at the head of a nearly-full disk's queue starved
+                        // forty smaller jobs that would have fit.
+                        processed.push(await queue.release(job.id, leaseId, {
+                            code: "insufficient_disk_space",
+                            message: `${room.detail} Streaming it to cloud instead was not possible: ${streamed.reason}.`
+                        }));
+                        refusedIds.add(job.id);
+                        continue;
+                    }
+                }
+                try {
+                    // Progress a person can watch: cumulative source bytes, written to the
+                    // job at most every 16 MB or 5% — enough to move visibly on a 1.6 GB
+                    // seal, cheap enough to be nothing on a 7 KB one. Per job, so several
+                    // concurrent seals each report their own.
+                    const totalBytes = await stat(job.sourcePath).then((s) => s.size).catch(() => 0);
+                    let lastMark = 0;
+                    const step = Math.max(16 * 1024 * 1024, Math.floor(totalBytes / 20));
+                    const onProgress = totalBytes > 0 ? (bytesRead) => {
+                        if (bytesRead - lastMark < step && bytesRead < totalBytes)
+                            return;
+                        lastMark = bytesRead;
+                        void queue.progress(job.id, leaseId, bytesRead, totalBytes).catch(() => { });
+                    } : undefined;
+                    const record = await archiveFile(dataDir, job.sourcePath, phrase, job.agent, { onProgress, delta: options.delta !== false });
+                    processed.push(await queue.complete(job.id, leaseId, { archiveId: record.id, ciphertextSha256: record.cipher.ciphertextSha256, bytes: record.source.bytes, deduplicated: record.deduplicated }));
+                }
+                catch (error) {
+                    try {
+                        processed.push(await queue.fail(job.id, leaseId, summarize(error)));
+                    }
+                    catch (leaseError) {
+                        processed.push({ ...job, lastError: summarize(leaseError) });
+                    }
+                }
+            }
+            finally {
+                // Released whatever happened: a completed seal's bytes are now real and
+                // measured by statfs, a refused or failed one never spent them.
+                if (guarded)
+                    reservations.release(job.id);
+            }
+        }
+    }
+    await Promise.all(Array.from({ length: concurrency }, (_, index) => work(concurrency > 1 ? `${owner}#${index + 1}` : owner)));
+    return processed;
+}

package/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "sealkeep",
+  "version": "0.5.0",
+  "type": "module",
+  "description": "Sealkeep by SPALA AI \u2014 your AI coding-agent history, sealed, searchable, and shared across your machines.",
+  "license": "SEE LICENSE IN LICENSE",
+  "engines": {
+    "node": ">=22.6.0"
+  },
+  "os": [
+    "darwin",
+    "linux",
+    "win32"
+  ],
+  "bin": {
+    "sealkeep": "dist/src/cli.js",
+    "vaultline": "dist/src/cli.js"
+  },
+  "files": [
+    "LICENSE",
+    "dist",
+    "web",
+    "README.md",
+    "ARCHITECTURE.md",
+    "CONTROL_PLANE.md",
+    "THREAT_MODEL.md",
+    "THIRD_PARTY.md",
+    "CHANGELOG.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.check.json",
+    "prepack": "npm run build",
+    "release:manifest": "node scripts/release.mjs",
+    "test": "node --import tsx --test \"test/**/*.test.ts\" \"packages/*/test/*.test.ts\"",
+    "vaultline": "tsx src/cli.ts",
+    "mcp": "tsx src/mcp.ts",
+    "api": "tsx src/cli.ts api",
+    "control-plane": "tsx src/control-plane-cli.ts",
+    "dashboard": "tsx src/dashboard-cli.ts",
+    "build:site": "node tools/build-site.mjs",
+    "bundle:site": "node tools/build-site.mjs && cd dist/site && zip -q -X ../site.zip index.html && cd - >/dev/null && echo \"dist/site.zip ready ($(wc -c < dist/site.zip) bytes)\""
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.2",
+    "tsx": "^4.19.2",
+    "typescript": "^5.7.2",
+    "zod": "^4.3.6"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.30.0",
+    "zod": "^4.4.3"
+  },
+  "keywords": [
+    "claude",
+    "codex",
+    "ai",
+    "agent",
+    "archive",
+    "encryption",
+    "backup",
+    "sessions"
+  ],
+  "homepage": "https://sealkeep.spala.ai"
+}