sealkeep 0.8.1 → 0.11.0

package/dist/src/queue.js

@@ -1,8 +1,162 @@
-import { link, mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
+import { link, mkdir, readdir, readFile, rename, rm, rmdir, stat, writeFile } from "node:fs/promises";
 import { createHash, randomUUID } from "node:crypto";
 import { join, resolve } from "node:path";
+import { classifyDaemonProcessGeneration, daemonProcessIdentity } from "./daemon-lease.js";
 import { fail } from "./errors.js";
 const JOB_VERSION = 1;
+/**
+ * The daemon worker and local API intentionally own separate `ArchiveQueue`
+ * instances. Keep their read snapshot at module scope so a status/dashboard
+ * burst still pays for one mature-queue walk, not one walk per object.
+ *
+ * This is a bounded cache because tests and embedded callers can construct
+ * queues for many short-lived roots in one process. Production ordinarily has
+ * one. An active read is never evicted.
+ */
+const sharedQueueReads = new Map();
+const MAX_SHARED_QUEUE_ROOTS = 32;
+// A mature startup scan can revisit thousands of already-settled rows. Even
+// when every stat/read is hot in the filesystem cache, a chain of immediately
+// fulfilled promises remains one microtask turn and can keep the HTTP poll
+// phase from accepting a local dashboard request. Yield on a fixed work bound,
+// not a time measurement: responsiveness must not depend on disk speed.
+const QUEUE_EVENT_LOOP_YIELD_EVERY = 32;
+const CANONICAL_JOB_FILE = /^([a-f0-9]{64})\.json$/;
+const LEGACY_LOCK_STALE_MS = 30_000;
+const DEFAULT_LOCK_ACQUIRE_MS = 30_000;
+let currentQueueProcessIdentity;
+function queueWorkYield(position) {
+    if ((position + 1) % QUEUE_EVENT_LOOP_YIELD_EVERY !== 0)
+        return undefined;
+    return new Promise((ready) => setImmediate(ready));
+}
+function trimSharedQueueReads(protectedKey) {
+    if (sharedQueueReads.size <= MAX_SHARED_QUEUE_ROOTS)
+        return;
+    for (const [candidate, value] of sharedQueueReads) {
+        if (candidate === protectedKey || value.inFlight || value.activeReaders > 0)
+            continue;
+        sharedQueueReads.delete(candidate);
+        if (sharedQueueReads.size <= MAX_SHARED_QUEUE_ROOTS)
+            break;
+    }
+}
+function sharedQueueReadState(root) {
+    const key = resolve(root);
+    let state = sharedQueueReads.get(key);
+    if (!state) {
+        state = { localRevision: 0, activeReaders: 0 };
+        sharedQueueReads.set(key, state);
+    }
+    else {
+        // Map insertion order doubles as a tiny LRU without another timer/handle.
+        sharedQueueReads.delete(key);
+        sharedQueueReads.set(key, state);
+    }
+    trimSharedQueueReads(key);
+    return state;
+}
+function invalidateSharedQueueRead(root) {
+    const state = sharedQueueReads.get(resolve(root));
+    if (!state)
+        return;
+    state.localRevision += 1;
+    state.cached = undefined;
+}
+function validTimestamp(value) {
+    return typeof value === "string" && Number.isFinite(Date.parse(value));
+}
+function validJob(value, expectedId) {
+    if (!value || typeof value !== "object")
+        return false;
+    const job = value;
+    if (job.version !== JOB_VERSION || job.id !== expectedId
+        || !CANONICAL_JOB_FILE.test(`${job.id}.json`)
+        || !["ready", "leased", "done", "failed"].includes(job.status)
+        || typeof job.agent !== "string" || !job.agent
+        || typeof job.sourcePath !== "string" || !job.sourcePath
+        || typeof job.event !== "string" || !job.event
+        || !Number.isSafeInteger(job.sourceBytes) || (job.sourceBytes ?? -1) < 0
+        || !validTimestamp(job.sourceModifiedAt)
+        || !validTimestamp(job.enqueuedAt)
+        || !validTimestamp(job.updatedAt)
+        || !Number.isSafeInteger(job.attempts) || (job.attempts ?? -1) < 0
+        || !validTimestamp(job.nextAttemptAt)
+        || (job.sessionId !== undefined && typeof job.sessionId !== "string")
+        || (job.supersededBy !== undefined && typeof job.supersededBy !== "string"))
+        return false;
+    if (job.lease !== undefined && (typeof job.lease.id !== "string" || !job.lease.id
+        || typeof job.lease.owner !== "string" || !job.lease.owner
+        || !validTimestamp(job.lease.acquiredAt) || !validTimestamp(job.lease.expiresAt)))
+        return false;
+    if (job.progress !== undefined && (!Number.isFinite(job.progress.bytes) || job.progress.bytes < 0
+        || !Number.isFinite(job.progress.of) || job.progress.of < 0))
+        return false;
+    if (job.lastError !== undefined && (typeof job.lastError.code !== "string" || typeof job.lastError.message !== "string"))
+        return false;
+    if (job.result !== undefined && (typeof job.result.archiveId !== "string" || !job.result.archiveId
+        || typeof job.result.ciphertextSha256 !== "string" || !job.result.ciphertextSha256
+        || !Number.isFinite(job.result.bytes) || job.result.bytes < 0
+        || !validTimestamp(job.result.completedAt)
+        || (job.result.deduplicated !== undefined && typeof job.result.deduplicated !== "boolean")))
+        return false;
+    return true;
+}
+function parseJobLockOwner(raw) {
+    if (!raw)
+        return null;
+    try {
+        const value = JSON.parse(raw);
+        if (value.version !== 1 || !Number.isSafeInteger(value.pid) || (value.pid ?? 0) <= 0
+            || typeof value.processIdentity !== "string" || !value.processIdentity || value.processIdentity.length > 512
+            || typeof value.instanceId !== "string" || !/^[a-f0-9-]{36}$/.test(value.instanceId)
+            || !validTimestamp(value.createdAt))
+            return null;
+        return value;
+    }
+    catch {
+        return null;
+    }
+}
+async function ownProcessIdentity() {
+    if (!currentQueueProcessIdentity) {
+        const lookup = daemonProcessIdentity(process.pid).then((current) => {
+            if (current.state !== "identified")
+                throw new Error("Sealkeep could not verify this process generation for the queue lock");
+            return current.identity;
+        });
+        currentQueueProcessIdentity = lookup;
+        void lookup.catch(() => {
+            // A transient procfs/ps failure must not poison every future queue
+            // transition in this long-lived process.
+            if (currentQueueProcessIdentity === lookup)
+                currentQueueProcessIdentity = undefined;
+        });
+    }
+    return currentQueueProcessIdentity;
+}
+async function processIdentityBefore(pid, deadline) {
+    const remaining = deadline - performance.now();
+    if (remaining <= 0)
+        return { state: "timeout" };
+    const lookup = (pid === process.pid
+        ? ownProcessIdentity().then((identity) => ({ state: "identified", identity }))
+        : daemonProcessIdentity(pid))
+        .catch(() => ({ state: "unknown" }));
+    let timer;
+    try {
+        return await Promise.race([
+            lookup,
+            new Promise((resolve) => {
+                timer = setTimeout(() => resolve({ state: "timeout" }), remaining);
+            }),
+        ]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
 export function queueDir(dataDir) { return join(dataDir, "queue"); }
 /**
  * Two lifecycle events describing the same unchanged transcript produce the same
@@ -23,6 +177,17 @@ export class ArchiveQueue {
     maxAttempts;
     backoffMs;
     maxBackoffMs;
+    lockAcquireMs;
+    /**
+     * `hasClaimable` already opened and parsed this record. Keep that answer for
+     * the immediately following claim so a mature queue does not enumerate and
+     * parse every completed job a second time merely to lease the work it just
+     * found. The hint is deliberately in-memory: the job file remains the
+     * durable source of truth and is revalidated before a lease is written.
+    */
+    claimableHint;
+    /** One startup reconciliation absorbs queue files written by older builds. */
+    reconciledBacklog = false;
     constructor(dataDir, options = {}) {
         this.root = queueDir(dataDir);
         this.now = options.now ?? Date.now;
@@ -30,14 +195,164 @@ export class ArchiveQueue {
         this.maxAttempts = options.maxAttempts ?? 5;
         this.backoffMs = options.backoffMs ?? 15_000;
         this.maxBackoffMs = options.maxBackoffMs ?? 30 * 60_000;
+        this.lockAcquireMs = Math.max(10, options.lockAcquireMs ?? DEFAULT_LOCK_ACQUIRE_MS);
     }
     path(id) { return join(this.root, `${id}.json`); }
+    lockPath(id) { return join(this.root, ".locks", `${id}.lock`); }
     stamp() { return new Date(this.now()).toISOString(); }
+    /**
+     * Serialises one job's read-check-write transition across every Sealkeep
+     * process on this machine. Atomic rename protects readers from partial JSON,
+     * but it is not compare-and-swap: without this lock, a delayed renewal can
+     * overwrite a newer worker's reclaimed lease, and a delayed progress write
+     * can resurrect a job after completion.
+     */
+    async withJobLock(id, work) {
+        // Queue-record transitions are one read-check-rename. A verified owner is
+        // never stolen, but callers must not disappear into an infinite retry loop
+        // if that owner is wedged or its process identity cannot be inspected.
+        const deadline = performance.now() + this.lockAcquireMs;
+        const lockDir = join(this.root, ".locks");
+        const lock = this.lockPath(id);
+        const ownerFile = join(lock, "owner");
+        const self = await processIdentityBefore(process.pid, deadline);
+        if (self.state !== "identified") {
+            return fail("internal", "Sealkeep could not verify this process generation before acquiring the queue lock", {
+                jobId: id,
+                reason: self.state === "timeout" ? "queue_lock_timeout" : "queue_lock_identity_unavailable",
+            });
+        }
+        const owner = {
+            version: 1,
+            pid: process.pid,
+            processIdentity: self.identity,
+            instanceId: randomUUID(),
+            createdAt: new Date().toISOString(),
+        };
+        const token = `${JSON.stringify(owner)}\n`;
+        await mkdir(lockDir, { recursive: true, mode: 0o700 });
+        for (;;) {
+            if (performance.now() >= deadline) {
+                return fail("internal", `Timed out waiting for queue job ${id} to become available`, {
+                    jobId: id,
+                    reason: "queue_lock_timeout",
+                });
+            }
+            const staging = join(lockDir, `.${id}.${randomUUID()}.pending`);
+            try {
+                // Build a complete, non-empty lock directory out of sight, then publish
+                // it with one rename. An observer can therefore never mistake the
+                // normal mkdir -> owner-file window for an abandoned malformed lock.
+                await mkdir(staging, { mode: 0o700 });
+                await writeFile(join(staging, "owner"), token, { mode: 0o600, flag: "wx" });
+                await rename(staging, lock);
+                break;
+            }
+            catch (error) {
+                await rm(join(staging, "owner"), { force: true }).catch(() => undefined);
+                await rmdir(staging).catch(() => undefined);
+                const code = error.code;
+                // macOS and Linux use different errors when rename meets an existing,
+                // non-empty directory.
+                if (code !== "EEXIST" && code !== "ENOTEMPTY")
+                    throw error;
+                const [held, owner] = await Promise.all([
+                    stat(lock).catch(() => null),
+                    readFile(ownerFile, "utf8").catch(() => null),
+                ]);
+                const parsedOwner = parseJobLockOwner(owner);
+                // Old queue builds wrote "pid uuid timestamp". Those locks are kept for
+                // one generous transition window when the PID is alive, then treated as
+                // stale: without a process generation, a recycled PID can never prove
+                // that it is the process which published this directory.
+                const legacyPid = parsedOwner ? NaN : Number.parseInt(owner?.split(/\s+/)[0] ?? "", 10);
+                const lookupPid = parsedOwner?.pid ?? legacyPid;
+                const current = Number.isSafeInteger(lookupPid) && lookupPid > 0
+                    ? await processIdentityBefore(lookupPid, deadline)
+                    : { state: "unknown" };
+                if (current.state === "timeout") {
+                    return fail("internal", `Timed out waiting for queue job ${id} to become available`, {
+                        jobId: id,
+                        reason: "queue_lock_timeout",
+                    });
+                }
+                const staleAge = !!held && Date.now() - held.mtimeMs > LEGACY_LOCK_STALE_MS;
+                const abandoned = current.state === "dead"
+                    || (parsedOwner && classifyDaemonProcessGeneration(current, parsedOwner.processIdentity) === "stale")
+                    || (!parsedOwner && staleAge);
+                if (abandoned) {
+                    // Move the exact abandoned directory aside atomically. The
+                    // quarantine name is derived from the directory identity, not from
+                    // a rescuer, so every process racing on this same stale lock chooses
+                    // the same destination. We intentionally leave the non-empty
+                    // quarantine in place: it fences a delayed observer from ever
+                    // renaming a successor lock after the first rescuer has moved on.
+                    const identity = held
+                        ? `${held.dev}:${held.ino}:${held.birthtimeMs}`
+                        : "missing";
+                    const fingerprint = createHash("sha256").update(identity).digest("hex").slice(0, 20);
+                    const quarantine = `${lock}.reclaimed-${fingerprint}`;
+                    try {
+                        if (!owner) {
+                            // A historical/corrupt empty lock needs content before it is
+                            // quarantined; otherwise POSIX rename could replace that empty
+                            // destination with a later live lock.
+                            await writeFile(join(lock, "malformed"), token, { mode: 0o600, flag: "wx" }).catch((markerError) => {
+                                if (markerError.code !== "EEXIST")
+                                    throw markerError;
+                            });
+                        }
+                        const [currentHeld, currentOwner] = await Promise.all([
+                            stat(lock).catch(() => null),
+                            readFile(ownerFile, "utf8").catch(() => null),
+                        ]);
+                        const stillSameDirectory = !!currentHeld
+                            && currentHeld.dev === held?.dev
+                            && currentHeld.ino === held?.ino
+                            && currentHeld.birthtimeMs === held?.birthtimeMs;
+                        if (stillSameDirectory && currentOwner === owner) {
+                            await rename(lock, quarantine);
+                            continue;
+                        }
+                    }
+                    catch (reclaimError) {
+                        const reclaimCode = reclaimError.code;
+                        if (reclaimCode !== "EEXIST" && reclaimCode !== "ENOTEMPTY" && reclaimCode !== "ENOENT")
+                            throw reclaimError;
+                    }
+                }
+                const remaining = deadline - performance.now();
+                if (remaining <= 0) {
+                    return fail("internal", `Timed out waiting for queue job ${id} to become available`, {
+                        jobId: id,
+                        reason: "queue_lock_timeout",
+                    });
+                }
+                await new Promise((resolve) => setTimeout(resolve, Math.min(10, remaining)));
+            }
+        }
+        try {
+            return await work();
+        }
+        finally {
+            // Only the holder that wrote this token may release this directory.
+            // rmdir is atomic and refuses a non-empty/replaced directory.
+            const current = await readFile(ownerFile, "utf8").catch(() => null);
+            if (current === token) {
+                await rm(ownerFile, { force: true }).catch(() => undefined);
+                await rmdir(lock).catch(() => undefined);
+            }
+        }
+    }
     async writeAtomic(job) {
         const target = this.path(job.id);
         const temp = `${target}.${randomUUID()}.tmp`;
         await writeFile(temp, JSON.stringify(job, null, 2) + "\n", { mode: 0o600 });
         await rename(temp, target);
+        // Invalidate every queue object in this process before the mutation's
+        // promise resolves. Other processes observe the atomic rename through the
+        // directory generation checked by `readJobsSnapshot`.
+        invalidateSharedQueueRead(this.root);
     }
     /** Creates the file only if the id is unseen; the content is complete before it becomes visible. */
     async createExclusive(job) {
@@ -46,6 +361,7 @@ export class ArchiveQueue {
         await writeFile(temp, JSON.stringify(job, null, 2) + "\n", { mode: 0o600 });
         try {
             await link(temp, target);
+            invalidateSharedQueueRead(this.root);
             return true;
         }
         catch (error) {
@@ -69,47 +385,217 @@ export class ArchiveQueue {
         const job = await this.get(id);
         return job ?? fail("queue_job_not_found", `No queued job with id ${id}`, { jobId: id });
     }
-    async list(filter) {
+    async readQueueDirectory() {
+        return readdir(this.root);
+    }
+    async readQueueFile(name) {
+        return readFile(join(this.root, name), "utf8");
+    }
+    async scanJobs() {
         let names;
         try {
-            names = await readdir(this.root);
+            names = await this.readQueueDirectory();
         }
-        catch {
-            return [];
+        catch (error) {
+            // A genuinely absent root is a complete empty queue. Permission, file
+            // descriptor and I/O failures are not: caching those as empty can strand
+            // already-durable work forever when no later write changes the directory.
+            return { jobs: [], complete: error.code === "ENOENT" };
         }
         const jobs = [];
-        for (const name of names) {
-            if (!name.endsWith(".json"))
-                continue;
-            try {
-                jobs.push(JSON.parse(await readFile(join(this.root, name), "utf8")));
+        let complete = true;
+        for (const [position, name] of names.entries()) {
+            const canonical = CANONICAL_JOB_FILE.exec(name);
+            if (canonical) {
+                try {
+                    const parsed = JSON.parse(await this.readQueueFile(name));
+                    if (validJob(parsed, canonical[1]))
+                        jobs.push(parsed);
+                    else
+                        complete = false;
+                }
+                catch {
+                    // Return healthy rows for availability, but never let this partial
+                    // census become cached authority or evidence for the done fast path.
+                    complete = false;
+                }
             }
-            catch {
-                continue;
+            // Count every directory entry, including a damaged/stale one. A queue
+            // full of files we skip must remain just as cooperative as a healthy one.
+            await queueWorkYield(position);
+        }
+        return { jobs, complete };
+    }
+    /**
+     * Atomic job replacement changes the queue directory's mtime/ctime. Reading
+     * those at nanosecond precision gives all Sealkeep processes a cheap durable
+     * generation check: sequential dashboard endpoints can reuse parsed jobs,
+     * while a hook/worker in another process invalidates that snapshot merely by
+     * committing its ordinary rename.
+     */
+    async directoryGeneration() {
+        try {
+            const value = await stat(this.root, { bigint: true });
+            return [value.dev, value.ino, value.size, value.mtimeNs, value.ctimeNs].join(":");
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                return "missing";
+            // Do not keep a completed snapshot when this process cannot validate the
+            // directory. The changing nonce deliberately forces the next read back
+            // to disk while preserving the queue's historical damaged-file tolerance.
+            return `unverifiable:${randomUUID()}`;
+        }
+    }
+    startSharedScan(state) {
+        if (state.inFlight)
+            return state.inFlight;
+        const scan = (async () => {
+            const localRevision = state.localRevision;
+            const before = await this.directoryGeneration();
+            const result = await this.scanJobs();
+            const after = await this.directoryGeneration();
+            const stable = result.complete && before === after && localRevision === state.localRevision && !after.startsWith("unverifiable:");
+            const snapshot = { directoryGeneration: after, localRevision, jobs: result.jobs, stable };
+            if (stable)
+                state.cached = snapshot;
+            return snapshot;
+        })();
+        state.inFlight = scan;
+        void scan.finally(() => {
+            if (state.inFlight === scan)
+                state.inFlight = undefined;
+        }).catch(() => undefined);
+        return scan;
+    }
+    async readJobsSnapshot() {
+        const state = sharedQueueReadState(this.root);
+        state.activeReaders += 1;
+        try {
+            // A caller that arrived behind a write overlapping the previous scan gets
+            // one retry against the new generation. Under a continuously mutating
+            // worker we then return the atomically parsed (but deliberately uncached)
+            // view instead of making the local API wait forever for perfect quiescence.
+            for (let attempt = 0; attempt < 2; attempt += 1) {
+                const observed = await this.directoryGeneration();
+                const cached = state.cached;
+                if (cached?.stable
+                    && cached.directoryGeneration === observed
+                    && cached.localRevision === state.localRevision) {
+                    return cached.jobs;
+                }
+                const snapshot = await this.startSharedScan(state);
+                const current = await this.directoryGeneration();
+                if (snapshot.stable
+                    && snapshot.directoryGeneration === current
+                    && snapshot.localRevision === state.localRevision) {
+                    return snapshot.jobs;
+                }
+                if (attempt === 1)
+                    return snapshot.jobs;
             }
+            return [];
+        }
+        finally {
+            state.activeReaders -= 1;
+            trimSharedQueueReads();
         }
+    }
+    /**
+     * Returns rows only when one durable directory generation covered the whole
+     * scan. `enqueueMany` uses this as a read-only fast path for settled ids; an
+     * unstable snapshot is never evidence that a job already exists.
+     */
+    async stableJobsSnapshot() {
+        const state = sharedQueueReadState(this.root);
+        state.activeReaders += 1;
+        try {
+            const observed = await this.directoryGeneration();
+            const cached = state.cached;
+            if (cached?.stable
+                && cached.directoryGeneration === observed
+                && cached.localRevision === state.localRevision) {
+                return cached.jobs;
+            }
+            const snapshot = await this.startSharedScan(state);
+            const current = await this.directoryGeneration();
+            if (snapshot.stable
+                && snapshot.directoryGeneration === current
+                && snapshot.localRevision === state.localRevision) {
+                return snapshot.jobs;
+            }
+            return null;
+        }
+        finally {
+            state.activeReaders -= 1;
+            trimSharedQueueReads();
+        }
+    }
+    async list(filter) {
+        const jobs = await this.readJobsSnapshot();
         return jobs.filter((job) => !filter?.status || job.status === filter.status).sort((a, b) => a.enqueuedAt.localeCompare(b.enqueuedAt) || a.id.localeCompare(b.id));
     }
     async stats() {
         const jobs = await this.list();
         const count = (status) => jobs.filter((job) => job.status === status).length;
-        return { total: jobs.length, ready: count("ready"), leased: count("leased"), done: count("done"), failed: count("failed") };
+        const completed = jobs.filter((job) => job.status === "done" && !!job.result).length;
+        const superseded = jobs.filter((job) => job.status === "done" && !!job.supersededBy && !job.result).length;
+        return {
+            total: jobs.length,
+            ready: count("ready"),
+            leased: count("leased"),
+            // Kept for wire compatibility: `done` is every settled queue row. It is
+            // not an archive count because older snapshots settle by supersession.
+            done: count("done"),
+            completed,
+            superseded,
+            failed: count("failed"),
+        };
+    }
+    /**
+     * Answers the worker's startup question from the same generation-validated
+     * snapshot used by the local API. A mature vault can have thousands of
+     * completed job files; independently reading every one merely to learn that
+     * one ready job exists made preservation compete with the status UI.
+     *
+     * Expired leases count too: after a crash they are work the next worker can
+     * reclaim, even though a plain `stats().ready` check would call the queue
+     * empty and wait for discovery first.
+     */
+    async hasClaimable(eligible) {
+        const jobs = await this.readJobsSnapshot();
+        let retired = new Set();
+        if (!this.reconciledBacklog) {
+            const outcome = await this.supersedeMany(jobs, jobs);
+            retired = new Set([...outcome.values()].flat());
+            this.reconciledBacklog = true;
+        }
+        const at = this.now();
+        for (const job of jobs) {
+            if (!retired.has(job.id) && this.claimable(job, at) && (!eligible || eligible(job))) {
+                this.claimableHint = job.id;
+                return true;
+            }
+        }
+        return false;
     }
     /**
      * Whether `candidate` describes a later state of the transcript than `over`.
      *
      * A total order over snapshots of one file, so two hook processes enqueueing
      * at once reach the same verdict and the most advanced snapshot is never the
-     * one superseded. Bytes lead because a transcript that has grown has strictly
-     * more in it; mtime settles a same-length rewrite. Equal on all three means
-     * the same content address, which is the same job.
+     * one superseded. Modification time leads because compaction can make a later
+     * transcript smaller; byte length settles the rare same-time observation.
+     * Equal on both means the same content address, which is the same job.
      */
     advances(candidate, over) {
-        if (candidate.sourceBytes !== over.sourceBytes)
-            return candidate.sourceBytes > over.sourceBytes;
         const [later, earlier] = [Date.parse(candidate.sourceModifiedAt), Date.parse(over.sourceModifiedAt)];
         if (later !== earlier)
             return later > earlier;
+        // Compaction and truncation legitimately make a later transcript smaller.
+        // Size therefore breaks a same-mtime tie; it cannot define chronology.
+        if (candidate.sourceBytes !== over.sourceBytes)
+            return candidate.sourceBytes > over.sourceBytes;
         return candidate.id > over.id;
     }
     /**
@@ -125,26 +611,111 @@ export class ArchiveQueue {
      *
      * Nothing is lost by dropping the earlier job: the bytes it would have
      * archived are the bytes still on disk, and the surviving job archives those.
-     * Only `ready` jobs are retired — a leased job may already be mid-archive, and
-     * a done or failed job is the record of something that actually happened.
+     * Ready jobs and abandoned expired leases may be retired; an active lease may
+     * already be mid-archive, and a done or failed job records something that
+     * actually happened.
      */
-    async supersede(winner) {
-        // A dead-lettered job archives nothing, so it may not retire work either.
-        if (winner.status === "failed")
-            return [];
-        const retired = [];
-        for (const job of await this.list({ status: "ready" })) {
-            if (job.id === winner.id || job.agent !== winner.agent || job.sourcePath !== winner.sourcePath)
+    sourceKey(job) {
+        return `${job.agent}\0${job.sourcePath}`;
+    }
+    /**
+     * Retires older ready snapshots for a whole discovery pass with one queue read.
+     *
+     * `enqueue()` used to call `list()` once per discovered transcript. A startup
+     * scan of N transcripts against Q queue records therefore parsed N*Q JSON
+     * files before it could seal anything. All prospective jobs are made visible
+     * first, then this method indexes the queue once and applies the same
+     * source-local ordering rule.
+     *
+     * A hook in another process remains safe: job creation is still exclusive. A
+     * hook that lands before this snapshot is included here; one that lands after
+     * it sees every batch job in its own reconciliation. Re-read before writing so
+     * a job claimed by a worker since the snapshot is never knowingly retired.
+     */
+    async supersedeMany(winners, snapshots) {
+        const retiredByWinner = new Map();
+        const strongest = new Map();
+        for (const winner of winners) {
+            // A dead-lettered job archives nothing, so it may not retire work either.
+            if (winner.status === "failed")
                 continue;
-            if (!this.advances(winner, job))
+            const key = this.sourceKey(winner);
+            const previous = strongest.get(key);
+            if (!previous || this.advances(winner, previous))
+                strongest.set(key, winner);
+        }
+        if (strongest.size === 0)
+            return retiredByWinner;
+        const retireable = (job, at = this.now()) => job.status === "ready"
+            || (job.status === "leased" && (!job.lease || Date.parse(job.lease.expiresAt) <= at));
+        // An active lease may already be sealing the earlier snapshot and stays
+        // untouched. An EXPIRED lease is different: it is already claimable by a
+        // replacement worker, and leaving it behind after discovery has published
+        // a newer snapshot makes the old row look permanently in-flight. Retiring
+        // it under the same per-job lock is the enqueue equivalent of reclaiming
+        // it, without doing the obsolete multi-gigabyte read again.
+        for (const snapshot of snapshots ?? await this.list()) {
+            if (!retireable(snapshot))
                 continue;
-            await this.writeAtomic({ ...job, status: "done", updatedAt: this.stamp(), lease: undefined, supersededBy: winner.id });
-            retired.push(job.id);
+            const winner = strongest.get(this.sourceKey(snapshot));
+            if (!winner || snapshot.id === winner.id || !this.advances(winner, snapshot))
+                continue;
+            const retired = await this.withJobLock(snapshot.id, async () => {
+                const current = await this.get(snapshot.id);
+                if (!current || !retireable(current) || !this.advances(winner, current))
+                    return false;
+                await this.writeAtomic({ ...current, status: "done", updatedAt: this.stamp(), lease: undefined, supersededBy: winner.id });
+                return true;
+            });
+            if (retired) {
+                const ids = retiredByWinner.get(winner.id) ?? [];
+                ids.push(snapshot.id);
+                retiredByWinner.set(winner.id, ids);
+            }
         }
-        return retired;
+        return retiredByWinner;
     }
-    /** Records intent to archive a transcript. It never reads transcript contents. */
-    async enqueue(request) {
+    /**
+     * Settles obsolete observations after a worker archives the live path.
+     *
+     * Queue rows contain metadata and a path, not a private copy of those old
+     * bytes. If an older row was already leased when discovery observed a newer
+     * state, that worker reads the newer live state and its completion covers
+     * the newer ready row too. Active *other* leases remain untouched; they may
+     * already have captured their own bounded snapshot.
+     */
+    async settleCoveredSourceSnapshots(winner, observed) {
+        const observedAt = Date.parse(observed.modifiedAt);
+        if (!Number.isFinite(observedAt))
+            return;
+        const retireable = (job, at = this.now()) => job.status === "ready"
+            || (job.status === "leased" && (!job.lease || Date.parse(job.lease.expiresAt) <= at));
+        const covered = (job) => {
+            if (job.id === winner.id || this.sourceKey(job) !== this.sourceKey(winner))
+                return false;
+            const jobAt = Date.parse(job.sourceModifiedAt);
+            return jobAt < observedAt || (jobAt === observedAt && job.sourceBytes === observed.bytes);
+        };
+        for (const snapshot of await this.list()) {
+            if (!retireable(snapshot) || !covered(snapshot))
+                continue;
+            await this.withJobLock(snapshot.id, async () => {
+                const current = await this.get(snapshot.id);
+                if (!current || !retireable(current) || !covered(current))
+                    return;
+                await this.writeAtomic({
+                    ...current,
+                    status: "done",
+                    updatedAt: this.stamp(),
+                    lease: undefined,
+                    progress: undefined,
+                    lastError: undefined,
+                    supersededBy: winner.id,
+                });
+            });
+        }
+    }
+    async describe(request) {
         const absolute = resolve(request.sourcePath);
         let source;
         try {
@@ -155,18 +726,113 @@ export class ArchiveQueue {
         }
         if (!source.isFile())
             fail("source_unreadable", `Transcript is not a file: ${absolute}`, { sourcePath: absolute });
-        await mkdir(this.root, { recursive: true, mode: 0o700 });
         const id = jobId({ agent: request.agent, sourcePath: absolute, bytes: source.size, modifiedAtMs: source.mtimeMs });
         const timestamp = this.stamp();
-        const job = {
+        return {
             version: JOB_VERSION, id, status: "ready", agent: request.agent, sourcePath: absolute, event: request.event, sessionId: request.sessionId,
             sourceBytes: source.size, sourceModifiedAt: new Date(source.mtimeMs).toISOString(),
             enqueuedAt: timestamp, updatedAt: timestamp, attempts: 0, nextAttemptAt: timestamp
         };
+    }
+    async persist(job) {
         if (await this.createExclusive(job))
-            return { job, deduped: false, superseded: await this.supersede(job) };
-        const existing = await this.require(id);
-        return { job: existing, deduped: true, superseded: await this.supersede(existing) };
+            return { job, deduped: false };
+        return { job: await this.require(job.id), deduped: true };
+    }
+    /** Records intent to archive a transcript. It never reads transcript contents. */
+    async enqueue(request) {
+        const described = await this.describe(request);
+        await mkdir(this.root, { recursive: true, mode: 0o700 });
+        let persisted = await this.persist(described);
+        // A click in Sessions is a durable local override, not merely another
+        // lifecycle label. If an automatic hook already created this exact
+        // content-addressed row, preserve the one row but promote its intent so an
+        // exclusion cannot make the person's explicit request unclaimable.
+        if (request.event === "manual" && persisted.job.event !== "manual"
+            && (persisted.job.status === "ready" || persisted.job.status === "failed")) {
+            const promoted = await this.withJobLock(persisted.job.id, async () => {
+                const current = await this.require(persisted.job.id);
+                if (current.status !== "ready" && current.status !== "failed")
+                    return current;
+                if (current.event === "manual")
+                    return current;
+                const next = { ...current, event: "manual", updatedAt: this.stamp() };
+                await this.writeAtomic(next);
+                return next;
+            });
+            persisted = { ...persisted, job: promoted };
+        }
+        const retired = await this.supersedeMany([persisted.job]);
+        return { ...persisted, superseded: retired.get(persisted.job.id) ?? [] };
+    }
+    /**
+     * Batch form used by startup/rescan discovery. Individual unreadable sources
+     * are reported without aborting the rest of the pass, matching the watcher's
+     * previous per-file failure isolation.
+     */
+    async enqueueMany(requests) {
+        if (requests.length === 0)
+            return [];
+        const outcomes = new Array(requests.length);
+        const described = [];
+        for (const [index, request] of requests.entries()) {
+            try {
+                described.push({ index, request, job: await this.describe(request) });
+            }
+            catch (error) {
+                outcomes[index] = { ok: false, request, error };
+            }
+            await queueWorkYield(index);
+        }
+        if (described.length === 0)
+            return outcomes;
+        try {
+            await mkdir(this.root, { recursive: true, mode: 0o700 });
+        }
+        catch (error) {
+            for (const item of described)
+                outcomes[item.index] = { ok: false, request: item.request, error };
+            return outcomes;
+        }
+        // Most rows in a mature vault are immutable `done` records. A prior
+        // implementation nevertheless wrote a complete temp file, attempted an
+        // exclusive hard-link, removed the temp and reread the winner for every
+        // one on every startup. On a 1,300-row real queue beside an index rebuild,
+        // that kept even /health out of the event loop for minutes.
+        //
+        // Only a generation-validated snapshot enables this shortcut, and only a
+        // done row is reused: ready/failed rows can be removed by the UI and leased
+        // rows can change ownership, so they retain the exclusive-create/re-read
+        // path. An id absent from the snapshot also retains that path, preserving
+        // convergence with a hook process that creates it after the snapshot.
+        const stable = await this.stableJobsSnapshot();
+        const settled = new Map((stable ?? [])
+            .filter((job) => job.status === "done")
+            .map((job) => [job.id, job]));
+        const persisted = [];
+        for (const [position, item] of described.entries()) {
+            try {
+                const known = settled.get(item.job.id);
+                persisted.push(known
+                    ? { ...item, job: known, deduped: true }
+                    : { ...item, ...await this.persist(item.job) });
+            }
+            catch (error) {
+                outcomes[item.index] = { ok: false, request: item.request, error };
+            }
+            await queueWorkYield(position);
+        }
+        const retired = await this.supersedeMany(persisted.map((item) => item.job));
+        const reported = new Set();
+        for (const item of persisted) {
+            // Duplicate requests in one batch describe one content-addressed job. As
+            // with sequential enqueue calls, report its retirements once, not once per
+            // duplicate lifecycle event.
+            const superseded = reported.has(item.job.id) ? [] : retired.get(item.job.id) ?? [];
+            reported.add(item.job.id);
+            outcomes[item.index] = { ok: true, request: item.request, job: item.job, deduped: item.deduped, superseded };
+        }
+        return outcomes;
     }
     claimable(job, at) {
         if (job.status === "ready")
@@ -187,40 +853,120 @@ export class ArchiveQueue {
      * several workers against the same instance, and two claims listing the
      * directory at the same moment would pick the same candidate and both write
      * a lease on it — the loser then failing its own job with a lease error.
-     * Serialising costs nothing next to the seal that follows, and cross-process
-     * claims keep the protection they always had: whoever renames second wins,
-     * and archiving is idempotent by source bytes.
+     * Serialising costs nothing next to the seal that follows. The per-job
+     * filesystem lock inside `claimNext` extends that same exclusivity across
+     * hook, daemon and manual-command processes; atomic rename alone cannot act
+     * as compare-and-swap and previously let both callers believe they won.
      */
-    async claim(owner, excluding) {
-        const turn = this.claimTurn.then(() => this.claimNext(owner, excluding));
+    async claim(owner, excluding, eligible) {
+        const turn = this.claimTurn.then(() => this.claimNext(owner, excluding, eligible));
         this.claimTurn = turn.then(() => undefined, () => undefined);
         return turn;
     }
-    async claimNext(owner, excluding) {
-        const at = this.now();
-        const candidate = (await this.list()).find((job) => this.claimable(job, at) && !excluding?.has(job.id));
-        if (!candidate)
-            return null;
-        const leased = {
-            ...candidate, status: "leased", attempts: candidate.attempts + 1, updatedAt: new Date(at).toISOString(),
-            lease: { id: randomUUID(), owner, acquiredAt: new Date(at).toISOString(), expiresAt: new Date(at + this.leaseMs).toISOString() }
+    async claimNext(owner, excluding, eligible) {
+        const acceptable = (job, at = this.now()) => this.claimable(job, at) && !excluding?.has(job.id) && (!eligible || eligible(job));
+        let candidates;
+        let retired = new Set();
+        if (!this.reconciledBacklog) {
+            candidates = await this.list();
+            const outcome = await this.supersedeMany(candidates, candidates);
+            retired = new Set([...outcome.values()].flat());
+            this.reconciledBacklog = true;
+        }
+        const hintedId = this.claimableHint;
+        this.claimableHint = undefined;
+        const tryClaim = async (candidate) => {
+            const leased = await this.withJobLock(candidate.id, async () => {
+                const current = await this.get(candidate.id);
+                const at = this.now();
+                if (!current || !acceptable(current, at))
+                    return null;
+                const next = {
+                    ...current, status: "leased", attempts: current.attempts + 1, updatedAt: new Date(at).toISOString(),
+                    lease: { id: randomUUID(), owner, acquiredAt: new Date(at).toISOString(), expiresAt: new Date(at + this.leaseMs).toISOString() }
+                };
+                await this.writeAtomic(next);
+                return next;
+            });
+            return leased;
         };
-        await this.writeAtomic(leased);
-        return leased;
+        // `hasClaimable` has already parsed this exact job. Try it before listing
+        // thousands of completed records; the lock re-reads it under exclusion so
+        // the hint is an optimization, never a trust boundary.
+        if (hintedId) {
+            const hinted = await this.get(hintedId);
+            if (hinted && acceptable(hinted)) {
+                const leased = await tryClaim(hinted);
+                if (leased)
+                    return leased;
+            }
+        }
+        for (const candidate of candidates ?? await this.list()) {
+            if (retired.has(candidate.id) || candidate.id === hintedId || !acceptable(candidate))
+                continue;
+            const leased = await tryClaim(candidate);
+            if (leased)
+                return leased;
+        }
+        return null;
     }
-    assertLease(job, leaseId) {
+    assertLeaseOwner(job, leaseId) {
         if (job.status !== "leased" || !job.lease)
             fail("queue_lease_invalid", `Job ${job.id} is not leased`, { jobId: job.id, status: job.status });
         if (job.lease.id !== leaseId)
             fail("queue_lease_invalid", `Lease does not match job ${job.id}`, { jobId: job.id });
-        if (Date.parse(job.lease.expiresAt) <= this.now())
-            fail("queue_lease_expired", `Lease for job ${job.id} expired at ${job.lease.expiresAt}`, { jobId: job.id, expiresAt: job.lease.expiresAt });
         return job;
     }
-    async complete(id, leaseId, result) {
-        const job = this.assertLease(await this.require(id), leaseId);
-        const done = { ...job, status: "done", updatedAt: this.stamp(), lease: undefined, lastError: undefined, result: { ...result, completedAt: this.stamp() } };
-        await this.writeAtomic(done);
+    assertLease(job, leaseId) {
+        // Expiry makes this record claimable; it does not silently change its id.
+        // Every mutation runs under the per-job lock. If a replacement claimed
+        // first, the id differs and this call is refused. If the id still matches,
+        // nobody owns a competing lease and the original worker may commit safely.
+        return this.assertLeaseOwner(job, leaseId);
+    }
+    /**
+     * How often a worker should prove it is still alive while holding a lease.
+     * Kept below one third of the lease, with a one-minute ceiling for the
+     * ordinary five-minute lease. Tests and small embedded queues can shorten
+     * the lease without having to guess a separate heartbeat setting.
+     */
+    leaseHeartbeatMs() {
+        return Math.max(10, Math.min(60_000, Math.floor(this.leaseMs / 3)));
+    }
+    /**
+     * Extends an unchanged lease without changing its owner, attempt count or
+     * original acquisition time. A delayed event loop may reach this write just
+     * after the deadline. The per-job lock makes that safe: if another worker
+     * reclaimed first, its new lease id wins and this renewal is refused; if the
+     * id is still ours, nobody took the work and renewal prevents a false orphan.
+     */
+    async renew(id, leaseId) {
+        return this.withJobLock(id, async () => {
+            const at = this.now();
+            const job = this.assertLeaseOwner(await this.require(id), leaseId);
+            const renewed = {
+                ...job,
+                updatedAt: new Date(at).toISOString(),
+                lease: { ...job.lease, expiresAt: new Date(at + this.leaseMs).toISOString() }
+            };
+            await this.writeAtomic(renewed);
+            return renewed;
+        });
+    }
+    async complete(id, leaseId, result, sourceSnapshot) {
+        const done = await this.withJobLock(id, async () => {
+            const job = this.assertLease(await this.require(id), leaseId);
+            const done = { ...job, status: "done", updatedAt: this.stamp(), lease: undefined, lastError: undefined, result: { ...result, completedAt: this.stamp() } };
+            await this.writeAtomic(done);
+            return done;
+        });
+        if (sourceSnapshot && Number.isFinite(sourceSnapshot.bytes) && Number.isFinite(Date.parse(sourceSnapshot.modifiedAt))) {
+            // A worker may claim yesterday's row while startup discovery publishes
+            // today's row for the same live path. The archive reads the current
+            // bounded snapshot, so settle every older/equal ready observation that
+            // this exact completion covered. A genuinely newer append survives.
+            await this.settleCoveredSourceSnapshots(done, sourceSnapshot);
+        }
         return done;
     }
     /**
@@ -234,53 +980,73 @@ export class ArchiveQueue {
      */
     /** Progress on the job being worked — visible, not merely true. */
     async progress(id, leaseId, bytes, of) {
-        const job = this.assertLease(await this.require(id), leaseId);
-        await this.writeAtomic({ ...job, progress: { bytes, of }, updatedAt: this.stamp() });
+        await this.withJobLock(id, async () => {
+            const at = this.now();
+            const job = this.assertLease(await this.require(id), leaseId);
+            // Progress is also proof of life. Extend in this same atomic document so
+            // the visible percentage can never land while leaving an older expiry
+            // behind for a competing claimant.
+            await this.writeAtomic({
+                ...job, progress: { bytes, of }, updatedAt: new Date(at).toISOString(),
+                lease: { ...job.lease, expiresAt: new Date(at + this.leaseMs).toISOString() }
+            });
+        });
     }
     /**
-     * Takes a waiting job back out of the queue. Only `ready` and `failed` jobs
-     * can go: a leased one is being sealed this moment and will finish or fail
-     * on its own, and a done one is not in the queue anymore — it is an archive,
-     * and removing that is a different, bigger decision made elsewhere.
+     * Takes a waiting job back out of the queue. Ready, failed, and expired-lease
+     * jobs can go. An unexpired lease is still owned by a worker and must finish
+     * or fail on its own; a done row is an archive and belongs to the vault.
      */
     async remove(id) {
-        const job = await this.require(id);
-        if (job.status === "leased")
-            fail("invalid_argument", "This one is being sealed right now — it finishes (or fails) on its own in a moment.", { jobId: id });
-        if (job.status === "done")
-            fail("invalid_argument", "Already sealed. What exists now is an archive; delete that from the vault, not from the queue.", { jobId: id });
-        await rm(this.path(id), { force: true });
-        return { ...job, status: "failed", updatedAt: this.stamp(), lastError: { code: "removed", message: "Taken out of the queue by hand" } };
+        return this.withJobLock(id, async () => {
+            const job = await this.require(id);
+            if (job.status === "leased" && job.lease && Date.parse(job.lease.expiresAt) > this.now()) {
+                fail("invalid_argument", "This one is being sealed right now — it finishes (or fails) on its own in a moment.", { jobId: id });
+            }
+            if (job.status === "done")
+                fail("invalid_argument", "Already sealed. What exists now is an archive; delete that from the vault, not from the queue.", { jobId: id });
+            await rm(this.path(id), { force: true });
+            invalidateSharedQueueRead(this.root);
+            return { ...job, status: "failed", updatedAt: this.stamp(), lastError: { code: "removed", message: "Taken out of the queue by hand" } };
+        });
     }
     async release(id, leaseId, reason) {
-        const job = this.assertLease(await this.require(id), leaseId);
-        const back = {
-            ...job, status: "ready", updatedAt: this.stamp(), lease: undefined,
-            attempts: Math.max(0, job.attempts - 1),
-            ...(reason ? { lastError: reason } : {})
-        };
-        await this.writeAtomic(back);
-        return back;
+        return this.withJobLock(id, async () => {
+            const job = this.assertLease(await this.require(id), leaseId);
+            const back = {
+                ...job, status: "ready", updatedAt: this.stamp(), lease: undefined,
+                attempts: Math.max(0, job.attempts - 1),
+                ...(reason ? { lastError: reason } : {})
+            };
+            await this.writeAtomic(back);
+            return back;
+        });
     }
     /** Reschedules with exponential backoff, or dead-letters once attempts are exhausted. */
     async fail(id, leaseId, error) {
-        const job = this.assertLease(await this.require(id), leaseId);
-        const exhausted = job.attempts >= this.maxAttempts;
-        const delay = Math.min(this.backoffMs * 2 ** Math.max(0, job.attempts - 1), this.maxBackoffMs);
-        const next = {
-            ...job, status: exhausted ? "failed" : "ready", updatedAt: this.stamp(), lease: undefined, lastError: error,
-            nextAttemptAt: exhausted ? job.nextAttemptAt : new Date(this.now() + delay).toISOString()
-        };
-        await this.writeAtomic(next);
-        return next;
+        return this.withJobLock(id, async () => {
+            const job = this.assertLease(await this.require(id), leaseId);
+            const exhausted = job.attempts >= this.maxAttempts;
+            const delay = Math.min(this.backoffMs * 2 ** Math.max(0, job.attempts - 1), this.maxBackoffMs);
+            const next = {
+                ...job, status: exhausted ? "failed" : "ready", updatedAt: this.stamp(), lease: undefined, lastError: error,
+                nextAttemptAt: exhausted ? job.nextAttemptAt : new Date(this.now() + delay).toISOString()
+            };
+            await this.writeAtomic(next);
+            return next;
+        });
     }
     /** Explicit operator action: a dead-lettered job never revives itself. */
     async retry(id) {
-        const job = await this.require(id);
-        if (job.status === "done")
-            fail("invalid_argument", `Job ${id} already completed`, { jobId: id });
-        const revived = { ...job, status: "ready", attempts: 0, updatedAt: this.stamp(), nextAttemptAt: this.stamp(), lease: undefined };
-        await this.writeAtomic(revived);
-        return revived;
+        return this.withJobLock(id, async () => {
+            const job = await this.require(id);
+            if (job.status === "done")
+                fail("invalid_argument", `Job ${id} already completed`, { jobId: id });
+            if (job.status === "leased")
+                fail("invalid_argument", `Job ${id} is being sealed and cannot be retried`, { jobId: id });
+            const revived = { ...job, status: "ready", attempts: 0, updatedAt: this.stamp(), nextAttemptAt: this.stamp(), lease: undefined };
+            await this.writeAtomic(revived);
+            return revived;
+        });
     }
 }