sealkeep 0.9.0 → 0.11.0

package/dist/src/chunk-store.js

@@ -4,17 +4,38 @@ import { join, resolve } from "node:path";
 import { fail } from "./errors.js";
 import { recordAudit } from "./audit.js";
 import { canonicalPhrase } from "./mnemonic.js";
-import { ADAPTER_VERSION, configuredRecipients, readConfig, writeRecord } from "./vault.js";
+import { isV2 } from "./types.js";
+import { ADAPTER_VERSION, configuredRecipients, mutateArchiveRecord, readConfig, teamSpaceOf } from "./vault.js";
+import { appendArchiveCopy } from "./archive-copies.js";
 import { uploadClientFromStore } from "./providers/index.js";
 import { acquireSpoolLock, createSpool, findSpoolForSource, shredSpool, unwrapSpoolKey, updateSpool } from "./spool.js";
-import { DEFAULT_CHUNK_BYTES, KEY_BYTES, StreamingSha256, sealChunksToSink, wrapAll, zeroize } from "../packages/vaultline-crypto/src/index.js";
+import { DEFAULT_CHUNK_BYTES, KEY_BYTES, StreamingSha256, sealChunksToSink, wrapAll, zeroize } from "../packages/sealkeep-crypto/src/index.js";
 import { frameObject } from "./cloud.js";
-import { aeadCipher, aeadDecipher } from "../packages/vaultline-crypto/src/aead.js";
-import { envVar } from "./env.js";
+import { aeadCipher, aeadDecipher } from "../packages/sealkeep-crypto/src/aead.js";
+import { STORAGE_PREFIX_DEFAULT } from "./branding.js";
+import { oneChunk } from "./byte-stream.js";
+import { paceBackgroundByteStream } from "./background-bandwidth.js";
 const SEAL_SUITE = "chacha20-poly1305";
 const sha256hex = (input) => createHash("sha256").update(input).digest("hex");
+async function waitForAbort(work, signal) {
+    if (!signal)
+        return work;
+    signal.throwIfAborted();
+    let cancel;
+    const aborted = new Promise((_resolve, reject) => {
+        cancel = () => reject(signal.reason instanceof Error ? signal.reason : new DOMException("The seal was cancelled", "AbortError"));
+        signal.addEventListener("abort", cancel, { once: true });
+    });
+    try {
+        return await Promise.race([work, aborted]);
+    }
+    finally {
+        if (cancel)
+            signal.removeEventListener("abort", cancel);
+    }
+}
 export const chunkObjectName = (index) => `chunk-${String(index).padStart(6, "0")}`;
-export const ENVELOPE_OBJECT = "envelope.vlmeta";
+export const ENVELOPE_OBJECT = "envelope.skmeta";
 /** One path segment a person can read in a bucket console, made key-safe. */
 function slug(raw, fallback) {
     const cleaned = raw.trim().replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^[-.]+|[-.]+$/g, "").slice(0, 60);
@@ -75,7 +96,7 @@ async function clientFor(dataDir, target) {
     // remoteStorage entry. The live drill on a real vault caught the gap:
     // gating on config.remoteStorage alone meant no production managed vault
     // could ever stream. A routed TARGET (multi-target vaults) overrides both.
-    const remoteStorage = (target ? { provider: target.provider, bucket: target.bucket ?? "vaultline-managed", prefix: target.prefix ?? "vaultline", region: target.region } : undefined)
+    const remoteStorage = (target ? { provider: target.provider, bucket: target.bucket ?? "vaultline-managed", prefix: target.prefix ?? STORAGE_PREFIX_DEFAULT, region: target.region } : undefined)
         ?? config.remoteStorage
         ?? await (async () => {
             const { cloudToken } = await import("./cloud.js");
@@ -96,7 +117,13 @@ async function clientFor(dataDir, target) {
         const { managedChunkClient } = await import("./managed-chunks.js");
         return { client: managedChunkClient(dataDir), remoteStorage: { provider: "vaultline", bucket: "vaultline-managed", prefix: "" } };
     }
-    const raw = await uploadClientFromStore(dataDir, config.vaultId, remoteStorage);
+    const resolvedTarget = target ?? { id: "primary", ...remoteStorage };
+    const { credentialConfigIdForTarget, endpointOverridesForResolvedTarget } = await import("./storage-targets.js");
+    const credentialId = await credentialConfigIdForTarget(dataDir, resolvedTarget);
+    if (!credentialId) {
+        fail("signer_not_configured", `Storage target "${resolvedTarget.id}" is not connected on this machine. Connect it in Settings before streaming.`);
+    }
+    const raw = await uploadClientFromStore(dataDir, credentialId, remoteStorage, await endpointOverridesForResolvedTarget(dataDir, resolvedTarget));
     const client = raw;
     if (typeof client.putObject !== "function" || typeof client.headObject !== "function" || typeof client.getObject !== "function" || typeof client.deleteObject !== "function") {
         fail("invalid_argument", `The ${remoteStorage.provider} client cannot store chunk folders`);
@@ -111,6 +138,7 @@ async function clientFor(dataDir, target) {
  * AND the provider's stored checksum, then skipped.
  */
 export async function sealToChunkFolder(dataDir, sourcePath, rawPhrase, agent, options = {}) {
+    options.signal?.throwIfAborted();
     const config = await readConfig(dataDir);
     if (config.storageMode === "plain") {
         fail("invalid_argument", "This vault stores archives as plain readable files; chunking them into a bucket would upload your transcripts unencrypted. Use a sealed vault for cloud archiving.");
@@ -119,10 +147,16 @@ export async function sealToChunkFolder(dataDir, sourcePath, rawPhrase, agent, o
     // byte limits are the rules; chunk streaming excludes Drive (single-object
     // by design). Injected clients (tests) bypass routing entirely.
     let routedTarget;
+    let destination;
+    let destinationFingerprint;
     if (!options.client) {
         const preSize = await stat(resolve(sourcePath)).then((info) => info.size).catch(() => 0);
-        const { chooseTarget } = await import("./storage-targets.js");
+        const { chooseTarget, destinationIdentityForTarget, storageDestinationFingerprint } = await import("./storage-targets.js");
         routedTarget = await chooseTarget(dataDir, { project: options.project ?? null, bytes: preSize, requireChunkCapable: true, targetId: options.targetId });
+        if (routedTarget.provider !== "vaultline") {
+            destination = await destinationIdentityForTarget(dataDir, routedTarget);
+            destinationFingerprint = storageDestinationFingerprint(destination);
+        }
     }
     const bundled = options.client ? null : await clientFor(dataDir, routedTarget);
     const remoteStorage = bundled?.remoteStorage ?? config.remoteStorage ?? { provider: "vaultline", bucket: "vaultline-managed", prefix: "" };
@@ -135,8 +169,10 @@ export async function sealToChunkFolder(dataDir, sourcePath, rawPhrase, agent, o
     const totalBytes = source.size;
     // Same override the local seal path honours, for the same reason: proving the
     // streamed layout works should not cost tens of megabytes of disk per run.
-    const chunkOverride = Number(envVar("CHUNK_BYTES"));
-    const chunkBytes = Number.isInteger(chunkOverride) && chunkOverride > 0 ? chunkOverride : DEFAULT_CHUNK_BYTES;
+    const chunkOverride = options.chunkBytes;
+    const chunkBytes = typeof chunkOverride === "number" && Number.isInteger(chunkOverride) && chunkOverride > 0
+        ? chunkOverride
+        : DEFAULT_CHUNK_BYTES;
     const totalChunks = Math.max(1, Math.ceil(totalBytes / chunkBytes));
     // Journal triage, chunk-layout edition. Same drain-the-stack shape as the
     // streaming path: resume the newest matching journal or abandon it with the
@@ -163,7 +199,7 @@ export async function sealToChunkFolder(dataDir, sourcePath, rawPhrase, agent, o
         }
         const lock = await acquireSpoolLock(dataDir, journal.archiveId, { now: options.now });
         try {
-            return await resumeChunkFolder({ dataDir, client, remoteStorage, targetId: routedTarget?.id, absolute, totalBytes, chunkBytes, totalChunks, phrase, agent, project: options.project ?? null, now: options.now }, journal);
+            return await resumeChunkFolder({ dataDir, client, remoteStorage, targetId: routedTarget?.id ?? options.targetId, destination, destinationFingerprint, absolute, totalBytes, chunkBytes, totalChunks, phrase, agent, project: options.project ?? null, projectKey: options.projectKey ?? null, teamRecipients: options.teamRecipients ?? [], now: options.now, signal: options.signal, maxBytesPerSecond: options.maxBytesPerSecond, onYield: options.onYield }, journal);
         }
         catch (error) {
             if (!(error instanceof Error) || !/cannot resume/i.test(error.message))
@@ -195,7 +231,7 @@ export async function sealToChunkFolder(dataDir, sourcePath, rawPhrase, agent, o
         naming, ...(naming === "hashed" ? { namingKey: folderNamingKey(phrase, storageScopeOf(remoteStorage)) } : {})
     });
     // Members of this project are wrapped in here, and nowhere else.
-    const recipients = configuredRecipients(config, phrase, { project: options.project ?? undefined });
+    const recipients = [...configuredRecipients(config, phrase, { project: options.project ?? undefined, projectKey: options.projectKey ?? undefined }), ...(options.teamRecipients ?? [])];
     const archiveKey = randomBytes(KEY_BYTES);
     const noncePrefix = randomBytes(4);
     const lock = await acquireSpoolLock(dataDir, archiveId, { now: options.now });
@@ -211,7 +247,7 @@ export async function sealToChunkFolder(dataDir, sourcePath, rawPhrase, agent, o
             hashState: { plaintext: "", stored: "" },
             providerState: { kind: "chunk-folder", folder, uploaded: 0 }
         }, { now: options.now });
-        return await runChunkSeal({ dataDir, client, remoteStorage, targetId: routedTarget?.id, absolute, totalBytes, chunkBytes, totalChunks, phrase, agent, project: options.project ?? null, now: options.now }, { archiveId, createdAt, folder, archiveKey, noncePrefix, project: options.project ?? null, startAt: 0, verifiedHeaders: [], reusedChunks: 0 });
+        return await runChunkSeal({ dataDir, client, remoteStorage, targetId: routedTarget?.id ?? options.targetId, destination, destinationFingerprint, absolute, totalBytes, chunkBytes, totalChunks, phrase, agent, project: options.project ?? null, projectKey: options.projectKey ?? null, teamRecipients: options.teamRecipients ?? [], now: options.now, signal: options.signal, maxBytesPerSecond: options.maxBytesPerSecond, onYield: options.onYield }, { archiveId, createdAt, folder, archiveKey, noncePrefix, project: options.project ?? null, startAt: 0, verifiedHeaders: [], reusedChunks: 0 });
     }
     finally {
         zeroize(archiveKey);
@@ -239,23 +275,29 @@ async function abandonFolderJournal(dataDir, client, journal, reason) {
  */
 async function runChunkSeal(ctx, args) {
     const config = await readConfig(ctx.dataDir);
-    const recipients = configuredRecipients(config, ctx.phrase, { project: ctx.project ?? undefined });
-    const { tokenize, indexableToken, tokenCapFor } = await import("./search.js");
-    const perChunkTokenCap = Math.max(4000, Math.floor(tokenCapFor(ctx.totalBytes) / 4));
+    const projectScope = teamSpaceOf(config, ctx.project, ctx.projectKey)?.spaceKey;
+    const recipients = [...configuredRecipients(config, ctx.phrase, { project: ctx.project ?? undefined, projectKey: ctx.projectKey ?? undefined }), ...ctx.teamRecipients];
+    const { createSealTokenCollector } = await import("./search.js");
     const hashers = { plaintext: new StreamingSha256(), stored: new StreamingSha256() };
     const headers = [];
     let storedBytes = 0;
     let heldAtMostBytes = 0;
-    // Uploads ride a small pool: the seal loop stays sequential (it is the CPU
-    // and the nonce discipline), but a sealed chunk does not wait for the wire
-    // before the next one seals. Four in flight bounds peak memory at four
-    // chunks and roughly quarters the wall clock of a lease-per-chunk upload.
-    const UPLOAD_CONCURRENCY = 4;
+    // User-triggered uploads ride a small pool: the seal loop stays sequential
+    // (it is the CPU and nonce discipline), while four in-flight PUTs overlap
+    // lease latency without holding more than four chunks. Automatic work is
+    // identified by its mandatory byte-rate ceiling and stays one-wide: disk
+    // pressure must not create a 4-chunk RAM/network burst in the background.
+    const uploadConcurrency = ctx.maxBytesPerSecond ? 1 : 4;
     const pending = new Map();
     const completedUploads = new Set();
     let watermark = args.startAt;
     let uploadError = null;
     let inFlightBytes = 0;
+    const paceUpload = async (body) => {
+        if (!ctx.maxBytesPerSecond)
+            return;
+        for await (const _ of paceBackgroundByteStream(ctx.dataDir, oneChunk(body), ctx.maxBytesPerSecond, { signal: ctx.signal })) { /* reservation only */ }
+    };
     // Journal writes are serialized: two concurrent read-modify-write updates
     // of the spool would lose one of them.
     let spoolChain = Promise.resolve();
@@ -267,7 +309,7 @@ async function runChunkSeal(ctx, args) {
     // Seal-time indexing: the seal holds every plaintext slice anyway, so the
     // tokens are collected here for free — the alternative was uploading the
     // archive and then downloading it back just to read it again.
-    const tokensByChunk = new Map();
+    const tokenCollector = createSealTokenCollector(ctx.totalBytes);
     const sink = {
         write: async (stored) => {
             const index = headers.length;
@@ -302,15 +344,16 @@ async function runChunkSeal(ctx, args) {
             await journal({ chunkHeaders: [...headers] });
             if (uploadError)
                 throw uploadError;
-            while (pending.size >= UPLOAD_CONCURRENCY) {
-                await Promise.race(pending.values());
+            while (pending.size >= uploadConcurrency) {
+                await waitForAbort(Promise.race(pending.values()), ctx.signal);
                 if (uploadError)
                     throw uploadError;
             }
             inFlightBytes += stored.length;
             heldAtMostBytes = Math.max(heldAtMostBytes, inFlightBytes);
             const task = (async () => {
-                await ctx.client.putObject(`${args.folder}/${chunkObjectName(index)}`, stored);
+                await paceUpload(stored);
+                await ctx.client.putObject(`${args.folder}/${chunkObjectName(index)}`, stored, { signal: ctx.signal });
                 completedUploads.add(index);
                 while (completedUploads.has(watermark))
                     watermark += 1;
@@ -330,27 +373,14 @@ async function runChunkSeal(ctx, args) {
         recipients, archiveId: args.archiveId, adapter: { agent: ctx.agent, version: ADAPTER_VERSION },
         suite: SEAL_SUITE, chunkBytes: ctx.chunkBytes, archiveKey: args.archiveKey, noncePrefix: args.noncePrefix,
         hashers, createdAt: args.createdAt,
-        onPlaintext: (chunkIndex, slice) => {
-            let bucket = tokensByChunk.get(chunkIndex);
-            if (!bucket) {
-                bucket = new Set();
-                tokensByChunk.set(chunkIndex, bucket);
-            }
-            // Filtered and budgeted at collection: junk (hex digests, base64
-            // runs) never takes a slot, and the budget scales with the archive —
-            // the flat 4000 starved everything past chunk zero of a 1.7 GB seal.
-            for (const token of tokenize(slice.toString("utf8"))) {
-                if (bucket.size >= perChunkTokenCap)
-                    break;
-                if (!indexableToken(token))
-                    continue;
-                bucket.add(token);
-            }
-        }
+        onPlaintext: tokenCollector.onPlaintext,
+        ...(ctx.onYield ? { onYield: ctx.onYield } : {}),
+        ...(ctx.signal ? { signal: ctx.signal } : {}),
     });
+    const tokensByChunk = tokenCollector.finish(ctx.totalChunks);
     // Drain the pool before anything downstream trusts the folder.
     while (pending.size > 0)
-        await Promise.race(pending.values());
+        await waitForAbort(Promise.race(pending.values()), ctx.signal);
     await spoolChain;
     if (uploadError)
         throw uploadError;
@@ -361,15 +391,16 @@ async function runChunkSeal(ctx, args) {
     // readable hierarchy back; a provider holding the bucket alone gets noise.
     const envelope = sealed.envelope;
     const identity = sealIdentity(args.archiveKey, args.archiveId, {
-        v: 1, project: args.project, sourcePath: ctx.absolute, agent: ctx.agent, createdAt: args.createdAt
+        v: 1, project: args.project, ...(ctx.projectKey ? { projectKey: ctx.projectKey } : {}), projectScope, sourcePath: ctx.absolute, agent: ctx.agent, createdAt: args.createdAt
     });
-    await ctx.client.putObject(`${args.folder}/${ENVELOPE_OBJECT}`, frameObject(envelope, identity));
+    const sidecar = frameObject(envelope, identity);
+    await ctx.client.putObject(`${args.folder}/${ENVELOPE_OBJECT}`, sidecar, { signal: ctx.signal });
     // Verification, per chunk, in the provider's own answers: every chunk
     // present at its recorded size. Byte-level trust is the chunk hash in the
     // envelope — restore proves it, AEAD enforces it.
     const problems = [];
     for (let index = 0; index < envelope.chunks.length; index += 1) {
-        const head = await ctx.client.headObject(`${args.folder}/${chunkObjectName(index)}`);
+        const head = await ctx.client.headObject(`${args.folder}/${chunkObjectName(index)}`, { signal: ctx.signal });
         if (!head.exists)
             problems.push(`chunk ${index} is not in the bucket`);
         else if (head.bytes !== envelope.chunks[index].storedBytes)
@@ -377,6 +408,13 @@ async function runChunkSeal(ctx, args) {
         if (problems.length >= 3)
             break;
     }
+    if (problems.length < 3) {
+        const sidecarHead = await ctx.client.headObject(`${args.folder}/${ENVELOPE_OBJECT}`, { signal: ctx.signal });
+        if (!sidecarHead.exists)
+            problems.push("the envelope sidecar is not in the bucket");
+        else if (sidecarHead.bytes !== sidecar.length)
+            problems.push(`the envelope sidecar is ${sidecarHead.bytes} bytes; ${sidecar.length} were sent`);
+    }
     if (problems.length > 0) {
         await recordAudit(ctx.dataDir, "upload.verify", "denied", { archiveId: args.archiveId, objectKey: args.folder, problems: problems.length });
         fail("ciphertext_integrity_failed", `Refusing to record the chunked archive as durable: ${problems.join("; ")}. The journal was kept; re-running resumes what is already there.`, { archiveId: args.archiveId, objectKey: args.folder });
@@ -386,32 +424,73 @@ async function runChunkSeal(ctx, args) {
         provider: ctx.remoteStorage.provider, bucket: ctx.remoteStorage.bucket, objectKey: args.folder,
         bytes: sealed.storedBytes, checksum: hashers.stored.digestBase64(), verifiedAt: now,
         layout: { kind: "chunks", chunkCount: envelope.chunks.length },
-        ...(ctx.targetId ? { targetId: ctx.targetId } : {})
+        ...(ctx.targetId ? { targetId: ctx.targetId } : {}),
+        ...(ctx.destination && ctx.destinationFingerprint
+            ? { destination: ctx.destination, destinationFingerprint: ctx.destinationFingerprint }
+            : {})
     };
-    const record = {
+    const completed = {
         version: 2, id: args.archiveId, createdAt: envelope.manifest.createdAt,
-        source: { path: ctx.absolute, agent: ctx.agent, bytes: ctx.totalBytes, sha256: sealed.originalSha256 },
+        source: { path: ctx.absolute, agent: ctx.agent, bytes: ctx.totalBytes, sha256: sealed.originalSha256, ...(ctx.project ? { project: ctx.project } : {}), ...(ctx.projectKey ? { projectKey: ctx.projectKey } : {}), ...(projectScope ? { projectScope } : {}) },
         cipher: { algorithm: envelope.suite, ciphertextSha256: sealed.ciphertextSha256, storedBytes: sealed.storedBytes, chunks: envelope.chunks.length },
         envelope,
-        objectPath: join(config.storage.root, `${args.archiveId}.vlarchive`),
+        objectPath: join(config.storage.root, `${args.archiveId}.skarchive`),
         remote,
         offloaded: { at: now, provider: ctx.remoteStorage.provider, bucket: ctx.remoteStorage.bucket, objectKey: args.folder }
     };
-    await writeRecord(config.storage.root, record);
+    const record = await mutateArchiveRecord(config.storage.root, args.archiveId, (current) => {
+        if (!isV2(current))
+            fail("invalid_argument", `Archive ${args.archiveId} uses format v1 and cannot receive a streamed copy`, { archiveId: args.archiveId });
+        // Completion owns the sealed archive facts, while the latest record owns
+        // any bookkeeping another process added. Most importantly, do not spread
+        // `completed.remote` over the current copy set: append/promote makes this
+        // verified destination the legacy primary and retains every older copy.
+        const { remote: _remote, copies: _copies, ...completedFacts } = completed;
+        return appendArchiveCopy({ ...current, ...completedFacts }, remote, { makePrimary: true });
+    }, { initial: completed });
     await recordAudit(ctx.dataDir, "archive.create", "allowed", { archiveId: args.archiveId, agent: ctx.agent, bytes: ctx.totalBytes, recipients: envelope.wrappedKeys.length, chunked: true, reused: args.reusedChunks });
     await recordAudit(ctx.dataDir, "upload.verify", "allowed", { archiveId: args.archiveId, objectKey: args.folder, bytes: sealed.storedBytes, provider: ctx.remoteStorage.provider, chunked: true });
     await shredSpool(ctx.dataDir, args.archiveId);
     // The tokens collected during the seal become the archive's index entry —
-    // search knows this session the moment the seal returns, and no machine
-    // ever downloads it back just to read it. Advisory by design: an index
-    // hiccup must never turn a durable archive into a failure.
+    // search knows this session the moment the seal returns, other machines
+    // learn it from the published index, and nobody downloads it back just to
+    // read it. First they are made durable beside the record: the merge below
+    // needs the whole index and the cloud, and when it failed (silently, for
+    // 2,300 seals in one week on a real vault) the tokens were lost with it and
+    // the archive was re-queued for a full decrypt. With the sidecar, a failed
+    // merge costs nothing: the next index pass merges it from the file. Both
+    // steps stay advisory — a hiccup must never turn a durable archive into a
+    // failure — but each is named, because "not indexed" used to be
+    // indistinguishable from "nothing there".
+    const failureName = (error) => {
+        const code = error?.code;
+        return typeof code === "string" ? code : error instanceof Error ? error.message.split("\n")[0] : String(error);
+    };
+    let preserved = false;
+    try {
+        const { writeArchiveTokenSidecar } = await import("./search.js");
+        await writeArchiveTokenSidecar(ctx.dataDir, ctx.phrase, args.archiveId, tokensByChunk);
+        preserved = true;
+    }
+    catch (error) {
+        const named = failureName(error);
+        process.emitWarning(`index tokens not preserved for ${args.archiveId.slice(0, 8)}: ${named}`, "SealkeepIndexDeferred");
+        await recordAudit(ctx.dataDir, "index.tokens", "denied", { archiveId: args.archiveId, reason: named }).catch(() => undefined);
+    }
     let indexed = false;
     try {
         const { indexArchiveTokens } = await import("./search.js");
-        await indexArchiveTokens(ctx.dataDir, ctx.phrase, record, tokensByChunk);
+        await indexArchiveTokens(ctx.dataDir, ctx.phrase, record, tokensByChunk, {
+            signal: ctx.signal,
+            maxBytesPerSecond: ctx.maxBytesPerSecond,
+        });
         indexed = true;
     }
-    catch { /* the next `index build` re-indexes this archive */ }
+    catch (error) {
+        const named = failureName(error);
+        process.emitWarning(`index merge deferred for ${args.archiveId.slice(0, 8)}: ${named}${preserved ? " — its tokens are preserved; the next index pass merges them without decrypting the archive" : ""}`, "SealkeepIndexDeferred");
+        await recordAudit(ctx.dataDir, "index.merge", "denied", { archiveId: args.archiveId, reason: named, preserved }).catch(() => undefined);
+    }
     // The account learns what each destination holds from the machines that
     // write to them — refresh the stamps now that this archive is recorded.
     {
@@ -442,7 +521,7 @@ async function resumeChunkFolder(ctx, journal) {
     let present = 0;
     const verifiedHeaders = [];
     while (present < journal.chunkHeaders.length) {
-        const head = await ctx.client.headObject(`${journal.objectKey}/${chunkObjectName(present)}`);
+        const head = await ctx.client.headObject(`${journal.objectKey}/${chunkObjectName(present)}`, { signal: ctx.signal });
         if (!head.exists)
             break;
         if (head.bytes !== journal.chunkHeaders[present].storedBytes) {
@@ -456,7 +535,7 @@ async function resumeChunkFolder(ctx, journal) {
         await recordAudit(ctx.dataDir, "upload.verify", "allowed", { archiveId: journal.archiveId, objectKey: journal.objectKey, resumedAtChunk: present });
         return await runChunkSeal(ctx, {
             archiveId: journal.archiveId, createdAt: journal.createdAt, folder: journal.objectKey,
-            archiveKey, noncePrefix, project: null, startAt: present, verifiedHeaders, reusedChunks: present
+            archiveKey, noncePrefix, project: ctx.project ?? null, startAt: present, verifiedHeaders, reusedChunks: present
         });
     }
     finally {
@@ -487,20 +566,37 @@ export function openIdentity(archiveKey, archiveId, blob) {
         return fail("invalid_argument", "The sidecar's identity does not open for this archive — wrong key, or a sidecar moved between folders.");
     }
 }
-/** Fetches a chunked remote back into one ciphertext buffer, hash-checked per chunk. */
-export async function fetchChunkFolder(client, remote, expected) {
+/** Fetches a chunked remote one bounded object at a time, hash-checking before yield. */
+export async function* fetchChunkFolderStream(client, remote, expected, options = {}) {
     if (remote.layout?.kind !== "chunks")
         fail("invalid_argument", "This remote is not a chunk folder");
-    const parts = [];
+    // The record's headers are the authority on what this archive is made of, not
+    // the bucket's own count. Trusting the remote count meant any chunk past the
+    // end of the recorded headers had no hash to check against and was folded
+    // into the ciphertext unverified — "every chunk is hash-checked" has to mean
+    // every chunk, or it is not a guarantee at all.
+    if (expected.chunks.length !== remote.layout.chunkCount) {
+        fail("ciphertext_integrity_failed", `This archive records ${expected.chunks.length} chunks but the bucket holds ${remote.layout.chunkCount}`, { objectKey: remote.objectKey });
+    }
     for (let index = 0; index < remote.layout.chunkCount; index += 1) {
-        const body = await client.getObject(`${remote.objectKey}/${chunkObjectName(index)}`);
+        options.signal?.throwIfAborted();
+        const body = await client.getObject(`${remote.objectKey}/${chunkObjectName(index)}`, { signal: options.signal });
         const header = expected.chunks[index];
-        if (header && sha256hex(body) !== header.sha256) {
+        if (!header || sha256hex(body) !== header.sha256) {
             fail("ciphertext_integrity_failed", `Chunk ${index} fetched from the bucket does not match its recorded hash`, { objectKey: remote.objectKey });
         }
-        parts.push(body);
+        yield body;
+    }
+}
+/** Compatibility API for callers that explicitly need the complete Buffer. */
+export async function fetchChunkFolder(client, remote, expected) {
+    const parts = [];
+    let bytes = 0;
+    for await (const part of fetchChunkFolderStream(client, remote, expected)) {
+        parts.push(part);
+        bytes += part.length;
     }
-    return Buffer.concat(parts);
+    return Buffer.concat(parts, bytes);
 }
 /** Deletes a chunked remote: every chunk object plus the envelope sidecar. */
 export async function deleteChunkFolder(client, remote) {