sealkeep 0.8.1 → 0.11.0

package/dist/src/providers/s3.js

@@ -1,6 +1,7 @@
 import { createHash } from "node:crypto";
 import { fail } from "../errors.js";
 import { presign } from "./sigv4.js";
+import { collectChunks, responseBodyChunks } from "../byte-stream.js";
 export const DEFAULT_PART_BYTES = 8 * 1024 * 1024;
 export const MIN_PART_BYTES = 5 * 1024 * 1024;
 const sha256b64 = (body) => createHash("sha256").update(body).digest("base64");
@@ -88,14 +89,16 @@ export class S3UploadClient {
     endpoint;
     credentials;
     partBytes;
+    fetchImpl;
     constructor(kind, endpoint, credentials, partBytes = DEFAULT_PART_BYTES, 
     /** S3 requires every part except the last to be at least 5 MiB. Local emulators and
      *  MinIO do not, so this is the deliberate opt-out for those targets only. */
-    allowSmallParts = false) {
+    allowSmallParts = false, fetchImpl = fetch) {
         this.kind = kind;
         this.endpoint = endpoint;
         this.credentials = credentials;
         this.partBytes = partBytes;
+        this.fetchImpl = fetchImpl;
         if (partBytes < MIN_PART_BYTES && !allowSmallParts)
             fail("invalid_argument", `S3 multipart parts must be at least ${MIN_PART_BYTES} bytes`, { partBytes });
         if (partBytes <= 0)
@@ -105,10 +108,10 @@ export class S3UploadClient {
         const { host, path } = target(this.endpoint, key);
         return presign({ method, host, port: this.endpoint.port, protocol: this.endpoint.protocol, path, region: this.endpoint.region, credentials: this.credentials, query, headers, expiresInSeconds });
     }
-    async put(key, body, query = {}) {
+    async put(key, body, query = {}, signal) {
         const checksum = sha256b64(body);
         const { url } = this.sign("PUT", key, query, { "x-amz-checksum-sha256": checksum });
-        const response = await fetch(url, { method: "PUT", body: new Uint8Array(body), headers: { "x-amz-checksum-sha256": checksum } });
+        const response = await this.fetchImpl(url, { method: "PUT", body: new Uint8Array(body), headers: { "x-amz-checksum-sha256": checksum }, signal });
         if (!response.ok)
             fail("internal", `Upload failed for ${key}: ${await readError(response)}`, { key, status: response.status });
         return { etag: (response.headers.get("etag") ?? "").replace(/"/g, ""), checksum };
@@ -119,44 +122,51 @@ export class S3UploadClient {
      * a plain checksummed PUT does everything multipart machinery did — which
      * is exactly why the chunk layout needs none of it.
      */
-    async putObject(key, body) {
-        const { checksum } = await this.put(key, body);
+    async putObject(key, body, options = {}) {
+        const { checksum } = await this.put(key, body, {}, options.signal);
         return { checksum };
     }
-    async headObject(key) {
+    async headObject(key, options = {}) {
         const { url } = this.sign("HEAD", key, {}, { "x-amz-checksum-mode": "ENABLED" });
-        const response = await fetch(url, { method: "HEAD", headers: { "x-amz-checksum-mode": "ENABLED" } });
+        const response = await this.fetchImpl(url, { method: "HEAD", headers: { "x-amz-checksum-mode": "ENABLED" }, signal: options.signal });
         if (response.status === 404)
             return { exists: false, bytes: 0 };
         if (!response.ok)
             fail("internal", `Could not read ${key}: ${response.status} ${response.statusText}`, { key, status: response.status });
         return { exists: true, bytes: Number(response.headers.get("content-length") ?? "0"), checksum: response.headers.get("x-amz-checksum-sha256") ?? undefined };
     }
-    async getObject(key) {
+    async getObject(key, options = {}) {
+        return collectChunks(await this.downloadKeyStream(key, options.signal), Number.MAX_SAFE_INTEGER);
+    }
+    /** Object-key streaming counterpart used by the background index store. */
+    async getObjectStream(key, options = {}) {
+        return this.downloadKeyStream(key, options.signal);
+    }
+    async downloadKeyStream(key, signal) {
         const { url } = this.sign("GET", key, {});
-        const response = await fetch(url);
+        const response = await this.fetchImpl(url, { signal });
         if (!response.ok)
             fail("internal", `Could not download ${key}: ${await readError(response)}`, { key, status: response.status });
-        return Buffer.from(await response.arrayBuffer());
+        return responseBodyChunks(response);
     }
     async deleteObject(key) {
         const { url } = this.sign("DELETE", key, {});
-        const response = await fetch(url, { method: "DELETE" });
+        const response = await this.fetchImpl(url, { method: "DELETE" });
         if (!response.ok && response.status !== 404)
             fail("internal", `Could not delete ${key}: ${await readError(response)}`, { key, status: response.status });
     }
-    async initiateMultipart(key) {
+    async initiateMultipart(key, signal) {
         const { url } = this.sign("POST", key, { uploads: "" });
-        const response = await fetch(url, { method: "POST" });
+        const response = await this.fetchImpl(url, { method: "POST", signal });
         if (!response.ok)
             fail("internal", `Could not start a multipart upload for ${key}: ${await readError(response)}`, { key });
         const uploadId = xmlValue(await response.text(), "UploadId");
         return uploadId ?? fail("internal", `Multipart initiation for ${key} returned no UploadId`, { key });
     }
-    async completeMultipart(key, uploadId, parts) {
+    async completeMultipart(key, uploadId, parts, signal) {
         const body = Buffer.from(`<CompleteMultipartUpload>${parts.map((part) => `<Part><PartNumber>${part.partNumber}</PartNumber><ETag>"${part.etag}"</ETag><ChecksumSHA256>${part.checksum}</ChecksumSHA256></Part>`).join("")}</CompleteMultipartUpload>`);
         const { url } = this.sign("POST", key, { uploadId });
-        const response = await fetch(url, { method: "POST", body: new Uint8Array(body) });
+        const response = await this.fetchImpl(url, { method: "POST", body: new Uint8Array(body), signal });
         if (!response.ok)
             fail("internal", `Could not complete the multipart upload for ${key}: ${await readError(response)}`, { key });
         const text = await response.text();
@@ -173,7 +183,7 @@ export class S3UploadClient {
      */
     async listParts(key, uploadId) {
         const { url } = this.sign("GET", key, { uploadId });
-        const response = await fetch(url);
+        const response = await this.fetchImpl(url);
         if (!response.ok)
             fail("internal", `Cannot resume this upload: the provider would not list its parts (${await readError(response)})`, { key, uploadId });
         const text = await response.text();
@@ -190,9 +200,9 @@ export class S3UploadClient {
         parts.sort((a, b) => a.partNumber - b.partNumber);
         return parts;
     }
-    async abortMultipart(key, uploadId) {
+    async abortMultipart(key, uploadId, signal) {
         const { url } = this.sign("DELETE", key, { uploadId });
-        await fetch(url, { method: "DELETE" }).catch(() => undefined);
+        await this.fetchImpl(url, { method: "DELETE", signal }).catch(() => undefined);
     }
     /** Uploads ciphertext and returns the checksum the provider confirmed. */
     async upload(lease, ciphertext) {
@@ -270,6 +280,7 @@ export class S3UploadClient {
         };
         try {
             for await (const chunk of chunks) {
+                options.signal?.throwIfAborted();
                 if (completionOnly && chunk.length > 0) {
                     fail("invalid_argument", "Cannot resume this upload with more data: the seeded parts already end in the object's final short part, so the only thing left is completion.", { key });
                 }
@@ -281,8 +292,8 @@ export class S3UploadClient {
                     const part = buffered.subarray(0, this.partBytes);
                     held = part.length < buffered.length ? [buffered.subarray(this.partBytes)] : [];
                     heldBytes = buffered.length - part.length;
-                    uploadId = uploadId ?? await this.initiateMultipart(key);
-                    const { etag, checksum } = await this.put(key, part, { partNumber: String(parts.length + 1), uploadId });
+                    uploadId = uploadId ?? await this.initiateMultipart(key, options.signal);
+                    const { etag, checksum } = await this.put(key, part, { partNumber: String(parts.length + 1), uploadId }, options.signal);
                     parts.push({ partNumber: parts.length + 1, etag, checksum, bytes: part.length });
                     await journal(uploadId);
                 }
@@ -292,15 +303,15 @@ export class S3UploadClient {
                 // The whole object fit inside one part: a single PUT stores it with a
                 // whole-object checksum, exactly like the buffered small-object path.
                 // (Never reached on resume — a seeded uploadId always exists there.)
-                const { checksum } = await this.put(key, tail);
+                const { checksum } = await this.put(key, tail, {}, options.signal);
                 return { remoteChecksum: checksum, bytes: total };
             }
             if (tail.length > 0 || parts.length === 0) {
-                const { etag, checksum } = await this.put(key, tail, { partNumber: String(parts.length + 1), uploadId });
+                const { etag, checksum } = await this.put(key, tail, { partNumber: String(parts.length + 1), uploadId }, options.signal);
                 parts.push({ partNumber: parts.length + 1, etag, checksum, bytes: tail.length });
                 await journal(uploadId);
             }
-            await this.completeMultipart(key, uploadId, parts);
+            await this.completeMultipart(key, uploadId, parts, options.signal);
             return { remoteChecksum: compositeChecksum(parts.map((part) => part.checksum)), bytes: total };
         }
         catch (error) {
@@ -317,12 +328,12 @@ export class S3UploadClient {
      * GET with the short-lived credential it was vended, exactly as it does for
      * the upload.
      */
-    async download(lease) {
-        const { url } = this.sign("GET", lease.objectKey, {});
-        const response = await fetch(url, { method: "GET" });
-        if (!response.ok)
-            fail("internal", `Could not read ${lease.objectKey}: ${await readError(response)}`, { key: lease.objectKey, status: response.status });
-        return Buffer.from(await response.arrayBuffer());
+    async download(lease, options = {}) {
+        return collectChunks(await this.downloadStream(lease, options), Number.MAX_SAFE_INTEGER);
+    }
+    /** The production restore/transfer door: response chunks flow straight to disk. */
+    async downloadStream(lease, options = {}) {
+        return this.downloadKeyStream(lease.objectKey, options.signal);
     }
     /**
      * Confirms the object exists remotely and reports what the provider stored.
@@ -335,7 +346,7 @@ export class S3UploadClient {
     async head(lease) {
         const mode = { "x-amz-checksum-mode": "ENABLED" };
         const { url } = this.sign("HEAD", lease.objectKey, {}, mode);
-        const response = await fetch(url, { method: "HEAD", headers: mode });
+        const response = await this.fetchImpl(url, { method: "HEAD", headers: mode });
         if (response.status === 404)
             return { exists: false, bytes: 0 };
         if (!response.ok)

package/dist/src/providers/safe-storage-fetch.d.ts

@@ -0,0 +1,12 @@
+import { type StorageAddressResolver } from "../storage-endpoint.js";
+export type StorageFetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
+/**
+ * Fetches only from one already-parsed storage origin. DNS policy executes in
+ * Undici's connector lookup, so the addresses it approves are the exact ones
+ * handed to the socket. The URL hostname remains untouched: signed Host,
+ * HTTPS SNI, and certificate hostname verification continue to use it.
+ */
+export declare function createPinnedStorageFetch(endpoint: string | URL, options?: {
+    allowPrivate?: boolean;
+    resolver?: StorageAddressResolver;
+}): StorageFetch;

package/dist/src/providers/safe-storage-fetch.js

@@ -0,0 +1,72 @@
+import { isIP } from "node:net";
+import { Agent, fetch as undiciFetch } from "undici";
+import { fail } from "../errors.js";
+import { allowedStorageAddresses, parseStorageEndpoint, systemStorageAddressResolver, } from "../storage-endpoint.js";
+function guardedLookup(allowPrivate, resolver) {
+    return (hostname, options, callback) => {
+        void allowedStorageAddresses(hostname, { allowPrivate, resolver }).then((answers) => {
+            const family = options.family === 4 || options.family === 6 ? options.family : 0;
+            const eligible = family ? answers.filter((answer) => answer.family === family) : answers;
+            if (eligible.length === 0) {
+                const error = new Error(`Storage endpoint ${hostname} has no address in the requested network family.`);
+                error.code = "EAI_ADDRFAMILY";
+                callback(error, "", 0);
+                return;
+            }
+            if (options.all)
+                callback(null, eligible);
+            else
+                callback(null, eligible[0].address, eligible[0].family);
+        }, (error) => callback(error instanceof Error ? error : new Error("Storage endpoint DNS validation failed."), "", 0));
+    };
+}
+const dispatchers = new WeakMap();
+function dispatcherFor(resolver, allowPrivate) {
+    let pair = dispatchers.get(resolver);
+    if (!pair) {
+        pair = {
+            public: new Agent({ connect: { lookup: guardedLookup(false, resolver) } }),
+            private: new Agent({ connect: { lookup: guardedLookup(true, resolver) } }),
+        };
+        dispatchers.set(resolver, pair);
+    }
+    return allowPrivate ? pair.private : pair.public;
+}
+/**
+ * Fetches only from one already-parsed storage origin. DNS policy executes in
+ * Undici's connector lookup, so the addresses it approves are the exact ones
+ * handed to the socket. The URL hostname remains untouched: signed Host,
+ * HTTPS SNI, and certificate hostname verification continue to use it.
+ */
+export function createPinnedStorageFetch(endpoint, options = {}) {
+    const expected = parseStorageEndpoint(String(endpoint));
+    const allowPrivate = options.allowPrivate === true;
+    const resolver = options.resolver ?? systemStorageAddressResolver;
+    const dispatcher = dispatcherFor(resolver, allowPrivate);
+    return async (input, init = {}) => {
+        let requested;
+        try {
+            requested = input instanceof URL ? new URL(input.href) : new URL(input);
+        }
+        catch {
+            return fail("invalid_argument", "Storage request did not contain a valid URL.");
+        }
+        if (requested.origin !== expected.origin) {
+            fail("invalid_argument", "Storage request tried to leave its configured endpoint; cross-origin storage requests are refused.");
+        }
+        // net.connect may bypass lookup for a literal. Apply the same rule before
+        // Undici can open that socket.
+        const literal = requested.hostname.toLowerCase().replace(/^\[|\]$/g, "");
+        if (isIP(literal))
+            await allowedStorageAddresses(literal, { allowPrivate, resolver });
+        const requestInit = {
+            ...init,
+            // A provider redirect can otherwise escape the origin check internally.
+            // Signed storage endpoints should already be final; fail closed on 3xx.
+            redirect: "manual",
+            dispatcher,
+        };
+        const response = await undiciFetch(requested, requestInit);
+        return response;
+    };
+}

package/dist/src/queue.d.ts

@@ -46,13 +46,29 @@ export type EnqueueRequest = {
     event: string;
     sessionId?: string;
 };
+export type EnqueueResult = {
+    job: ArchiveJob;
+    deduped: boolean;
+    superseded: string[];
+};
+export type BatchEnqueueOutcome = ({
+    ok: true;
+    request: EnqueueRequest;
+} & EnqueueResult) | {
+    ok: false;
+    request: EnqueueRequest;
+    error: unknown;
+};
 export type QueueOptions = {
     now?: () => number;
     leaseMs?: number;
     maxAttempts?: number;
     backoffMs?: number;
     maxBackoffMs?: number;
+    /** Real-time bound for one tiny queue-record state transition. */
+    lockAcquireMs?: number;
 };
+export type ClaimEligibility = (job: ArchiveJob) => boolean;
 export declare function queueDir(dataDir: string): string;
 /**
  * Two lifecycle events describing the same unchanged transcript produce the same
@@ -76,14 +92,53 @@ export declare class ArchiveQueue {
     private readonly maxAttempts;
     private readonly backoffMs;
     private readonly maxBackoffMs;
+    private readonly lockAcquireMs;
+    /**
+     * `hasClaimable` already opened and parsed this record. Keep that answer for
+     * the immediately following claim so a mature queue does not enumerate and
+     * parse every completed job a second time merely to lease the work it just
+     * found. The hint is deliberately in-memory: the job file remains the
+     * durable source of truth and is revalidated before a lease is written.
+    */
+    private claimableHint?;
+    /** One startup reconciliation absorbs queue files written by older builds. */
+    private reconciledBacklog;
     constructor(dataDir: string, options?: QueueOptions);
     private path;
+    private lockPath;
     private stamp;
+    /**
+     * Serialises one job's read-check-write transition across every Sealkeep
+     * process on this machine. Atomic rename protects readers from partial JSON,
+     * but it is not compare-and-swap: without this lock, a delayed renewal can
+     * overwrite a newer worker's reclaimed lease, and a delayed progress write
+     * can resurrect a job after completion.
+     */
+    private withJobLock;
     private writeAtomic;
     /** Creates the file only if the id is unseen; the content is complete before it becomes visible. */
     private createExclusive;
     get(id: string): Promise<ArchiveJob | null>;
     private require;
+    private readQueueDirectory;
+    private readQueueFile;
+    private scanJobs;
+    /**
+     * Atomic job replacement changes the queue directory's mtime/ctime. Reading
+     * those at nanosecond precision gives all Sealkeep processes a cheap durable
+     * generation check: sequential dashboard endpoints can reuse parsed jobs,
+     * while a hook/worker in another process invalidates that snapshot merely by
+     * committing its ordinary rename.
+     */
+    private directoryGeneration;
+    private startSharedScan;
+    private readJobsSnapshot;
+    /**
+     * Returns rows only when one durable directory generation covered the whole
+     * scan. `enqueueMany` uses this as a read-only fast path for settled ids; an
+     * unstable snapshot is never evidence that a job already exists.
+     */
+    private stableJobsSnapshot;
     list(filter?: {
         status?: JobStatus;
     }): Promise<ArchiveJob[]>;
@@ -92,16 +147,29 @@ export declare class ArchiveQueue {
         ready: number;
         leased: number;
         done: number;
+        completed: number;
+        superseded: number;
         failed: number;
     }>;
+    /**
+     * Answers the worker's startup question from the same generation-validated
+     * snapshot used by the local API. A mature vault can have thousands of
+     * completed job files; independently reading every one merely to learn that
+     * one ready job exists made preservation compete with the status UI.
+     *
+     * Expired leases count too: after a crash they are work the next worker can
+     * reclaim, even though a plain `stats().ready` check would call the queue
+     * empty and wait for discovery first.
+     */
+    hasClaimable(eligible?: ClaimEligibility): Promise<boolean>;
     /**
      * Whether `candidate` describes a later state of the transcript than `over`.
      *
      * A total order over snapshots of one file, so two hook processes enqueueing
      * at once reach the same verdict and the most advanced snapshot is never the
-     * one superseded. Bytes lead because a transcript that has grown has strictly
-     * more in it; mtime settles a same-length rewrite. Equal on all three means
-     * the same content address, which is the same job.
+     * one superseded. Modification time leads because compaction can make a later
+     * transcript smaller; byte length settles the rare same-time observation.
+     * Equal on both means the same content address, which is the same job.
      */
     private advances;
     /**
@@ -117,16 +185,46 @@ export declare class ArchiveQueue {
      *
      * Nothing is lost by dropping the earlier job: the bytes it would have
      * archived are the bytes still on disk, and the surviving job archives those.
-     * Only `ready` jobs are retired — a leased job may already be mid-archive, and
-     * a done or failed job is the record of something that actually happened.
+     * Ready jobs and abandoned expired leases may be retired; an active lease may
+     * already be mid-archive, and a done or failed job records something that
+     * actually happened.
      */
-    private supersede;
+    private sourceKey;
+    /**
+     * Retires older ready snapshots for a whole discovery pass with one queue read.
+     *
+     * `enqueue()` used to call `list()` once per discovered transcript. A startup
+     * scan of N transcripts against Q queue records therefore parsed N*Q JSON
+     * files before it could seal anything. All prospective jobs are made visible
+     * first, then this method indexes the queue once and applies the same
+     * source-local ordering rule.
+     *
+     * A hook in another process remains safe: job creation is still exclusive. A
+     * hook that lands before this snapshot is included here; one that lands after
+     * it sees every batch job in its own reconciliation. Re-read before writing so
+     * a job claimed by a worker since the snapshot is never knowingly retired.
+     */
+    private supersedeMany;
+    /**
+     * Settles obsolete observations after a worker archives the live path.
+     *
+     * Queue rows contain metadata and a path, not a private copy of those old
+     * bytes. If an older row was already leased when discovery observed a newer
+     * state, that worker reads the newer live state and its completion covers
+     * the newer ready row too. Active *other* leases remain untouched; they may
+     * already have captured their own bounded snapshot.
+     */
+    private settleCoveredSourceSnapshots;
+    private describe;
+    private persist;
     /** Records intent to archive a transcript. It never reads transcript contents. */
-    enqueue(request: EnqueueRequest): Promise<{
-        job: ArchiveJob;
-        deduped: boolean;
-        superseded: string[];
-    }>;
+    enqueue(request: EnqueueRequest): Promise<EnqueueResult>;
+    /**
+     * Batch form used by startup/rescan discovery. Individual unreadable sources
+     * are reported without aborting the rest of the pass, matching the watcher's
+     * previous per-file failure isolation.
+     */
+    enqueueMany(requests: readonly EnqueueRequest[]): Promise<BatchEnqueueOutcome[]>;
     private claimable;
     /** Claims from this instance run one at a time; see `claim`. */
     private claimTurn;
@@ -140,18 +238,38 @@ export declare class ArchiveQueue {
      * several workers against the same instance, and two claims listing the
      * directory at the same moment would pick the same candidate and both write
      * a lease on it — the loser then failing its own job with a lease error.
-     * Serialising costs nothing next to the seal that follows, and cross-process
-     * claims keep the protection they always had: whoever renames second wins,
-     * and archiving is idempotent by source bytes.
+     * Serialising costs nothing next to the seal that follows. The per-job
+     * filesystem lock inside `claimNext` extends that same exclusivity across
+     * hook, daemon and manual-command processes; atomic rename alone cannot act
+     * as compare-and-swap and previously let both callers believe they won.
      */
-    claim(owner: string, excluding?: ReadonlySet<string>): Promise<ArchiveJob | null>;
+    claim(owner: string, excluding?: ReadonlySet<string>, eligible?: ClaimEligibility): Promise<ArchiveJob | null>;
     private claimNext;
+    private assertLeaseOwner;
     private assertLease;
+    /**
+     * How often a worker should prove it is still alive while holding a lease.
+     * Kept below one third of the lease, with a one-minute ceiling for the
+     * ordinary five-minute lease. Tests and small embedded queues can shorten
+     * the lease without having to guess a separate heartbeat setting.
+     */
+    leaseHeartbeatMs(): number;
+    /**
+     * Extends an unchanged lease without changing its owner, attempt count or
+     * original acquisition time. A delayed event loop may reach this write just
+     * after the deadline. The per-job lock makes that safe: if another worker
+     * reclaimed first, its new lease id wins and this renewal is refused; if the
+     * id is still ours, nobody took the work and renewal prevents a false orphan.
+     */
+    renew(id: string, leaseId: string): Promise<ArchiveJob>;
     complete(id: string, leaseId: string, result: {
         archiveId: string;
         ciphertextSha256: string;
         bytes: number;
         deduplicated?: boolean;
+    }, sourceSnapshot?: {
+        bytes: number;
+        modifiedAt: string;
     }): Promise<ArchiveJob>;
     /**
      * Puts a claimed job back without spending an attempt.
@@ -165,10 +283,9 @@ export declare class ArchiveQueue {
     /** Progress on the job being worked — visible, not merely true. */
     progress(id: string, leaseId: string, bytes: number, of: number): Promise<void>;
     /**
-     * Takes a waiting job back out of the queue. Only `ready` and `failed` jobs
-     * can go: a leased one is being sealed this moment and will finish or fail
-     * on its own, and a done one is not in the queue anymore — it is an archive,
-     * and removing that is a different, bigger decision made elsewhere.
+     * Takes a waiting job back out of the queue. Ready, failed, and expired-lease
+     * jobs can go. An unexpired lease is still owned by a worker and must finish
+     * or fail on its own; a done row is an archive and belongs to the vault.
      */
     remove(id: string): Promise<ArchiveJob>;
     release(id: string, leaseId: string, reason?: {