sealkeep 0.5.0

package/dist/packages/vaultline-crypto/src/stream.js

@@ -0,0 +1,477 @@
+/**
+ * File-to-file sealing and opening, in constant memory.
+ *
+ * `envelope.ts` holds the whole plaintext, the whole compressed body and the
+ * whole ciphertext at once, so archiving a 4 GB agent rollout needed roughly
+ * 14.5 GB of resident memory. The archives these functions produce are the
+ * *same* format — a v2 envelope, gzip applied to the whole body before
+ * chunking — so anything written here opens with `decryptArchive`, and anything
+ * written by `encryptArchive` opens with `openArchiveToFile`. Only the memory
+ * profile changes.
+ *
+ * The one thing that stops a seal being a single streaming pass is the
+ * associated data: every chunk commits to the *total* chunk count, which is not
+ * known until the compressed body is complete. Rather than weaken that binding,
+ * the body is written once to scratch, measured, and then sealed. Two sequential
+ * streams, no format change, nothing larger than one chunk ever resident.
+ */
+import { createReadStream, createWriteStream } from "node:fs";
+import { open, rm, stat } from "node:fs/promises";
+import { pipeline } from "node:stream/promises";
+import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
+import { createHash, randomBytes, randomUUID } from "node:crypto";
+import { dirname, join } from "node:path";
+import { aeadCipher, aeadDecipher } from "./aead.js";
+import { CryptoError, unwrapArchiveKey, zeroize } from "./recipients.js";
+import { StreamingSha256 } from "./sha256-stream.js";
+import { wrapAll, validateEnvelope } from "./envelope.js";
+import { DEFAULT_CHUNK_BYTES, ENVELOPE_VERSION, KEY_BYTES, SUITES, TAG_BYTES, chunkAad, chunkNonce } from "./format.js";
+const sha256 = (input) => createHash("sha256").update(input).digest("hex");
+/**
+ * Both hash flavours answer in hex here; the branch exists so the seal loop can
+ * feed an injected resumable hasher and a fresh `createHash` identically. Note
+ * `StreamingSha256` survives its digest, so an injected instance can also be
+ * asked for base64 afterwards — the two output formats recorded for one seal.
+ */
+const digestHex = (hash) => hash instanceof StreamingSha256 ? hash.digestHex() : hash.digest("hex");
+/** `FileHandle.read` may come up short; a chunk boundary is not a suggestion. */
+async function readExactly(handle, into, length, position) {
+    let filled = 0;
+    while (filled < length) {
+        const { bytesRead } = await handle.read(into, filled, length - filled, position + filled);
+        if (bytesRead === 0)
+            throw new CryptoError("ciphertext_length_mismatch", `Expected ${length} bytes at offset ${position}; the file ended after ${filled}`);
+        filled += bytesRead;
+    }
+    return into.subarray(0, length);
+}
+/**
+ * Seals a byte range of one file into another, holding one chunk at a time.
+ *
+ * The destination is written but never renamed here: the manifest hashes are
+ * only proven at the very end, so publishing the result is the caller's job and
+ * must happen after this resolves. On failure the destination is removed.
+ */
+export async function sealArchiveToFile(source, destPath, options) {
+    const suite = options.suite ?? "chacha20-poly1305";
+    if (!SUITES.includes(suite))
+        throw new CryptoError("suite_unsupported", `Unsupported suite: ${suite}`);
+    const chunkBytes = options.chunkBytes ?? DEFAULT_CHUNK_BYTES;
+    if (!Number.isInteger(chunkBytes) || chunkBytes <= 0)
+        throw new CryptoError("invalid_chunk_size", "chunkBytes must be a positive integer");
+    const archiveId = options.archiveId ?? randomUUID();
+    const archiveKey = options.deterministic?.archiveKey ?? randomBytes(KEY_BYTES);
+    const noncePrefix = options.deterministic?.noncePrefix ?? randomBytes(4);
+    if (archiveKey.length !== KEY_BYTES)
+        throw new CryptoError("key_malformed", `An archive key must be ${KEY_BYTES} bytes`);
+    const compressed = (options.compression ?? "none") === "gzip";
+    const chunkCompressed = (options.compression ?? "none") === "gzip-chunk";
+    const start = source.start ?? 0;
+    const sourceBytes = (await stat(source.path)).size;
+    const end = Math.min(source.end ?? sourceBytes, sourceBytes);
+    if (start < 0 || end < start)
+        throw new CryptoError("invalid_range", `Range ${start}..${end} is not a range of ${source.path}`);
+    if (chunkCompressed) {
+        const dest0 = await open(destPath, "w", 0o600);
+        try {
+            const result = await sealChunksToSink(source, {
+                write: async (stored) => {
+                    const { bytesWritten } = await dest0.write(stored);
+                    if (bytesWritten !== stored.length) {
+                        throw new CryptoError("destination_write_failed", `Only ${bytesWritten} of ${stored.length} bytes reached ${destPath} — the volume is probably full`);
+                    }
+                }
+            }, { ...options, suite, chunkBytes, archiveId, archiveKey, noncePrefix });
+            await dest0.close();
+            if (!options.deterministic)
+                zeroize(archiveKey);
+            return result;
+        }
+        catch (error) {
+            await dest0.close();
+            if (!options.deterministic)
+                zeroize(archiveKey);
+            await rm(destPath, { force: true });
+            throw error;
+        }
+    }
+    const scratch = options.scratchDir ?? dirname(destPath);
+    const bodyPath = compressed ? join(scratch, `.vlbody-${randomUUID()}`) : source.path;
+    const originalHash = createHash("sha256");
+    let originalBytes = 0;
+    try {
+        if (compressed) {
+            // Pass one: the caller's bytes are hashed on the way past, so the source
+            // is read exactly once even though the body is produced separately.
+            await pipeline(createReadStream(source.path, { start, end: Math.max(start, end - 1) }), async function* (chunks) {
+                for await (const piece of chunks) {
+                    originalHash.update(piece);
+                    originalBytes += piece.length;
+                    options.onProgress?.(originalBytes);
+                    yield piece;
+                }
+            }, createGzip({ level: 6 }), createWriteStream(bodyPath, { mode: 0o600 }));
+        }
+        const bodyStart = compressed ? 0 : start;
+        const bodyBytes = compressed ? (await stat(bodyPath)).size : end - start;
+        const totalChunks = Math.max(1, Math.ceil(bodyBytes / chunkBytes));
+        const chunks = [];
+        const bodyHash = createHash("sha256");
+        const ciphertextHash = createHash("sha256");
+        let storedBytes = 0;
+        const buffer = Buffer.allocUnsafe(Math.min(chunkBytes, Math.max(bodyBytes, 1)));
+        const body = await open(bodyPath, "r");
+        const dest = await open(destPath, "w", 0o600);
+        try {
+            for (let index = 0; index < totalChunks; index += 1) {
+                const want = Math.max(0, Math.min(chunkBytes, bodyBytes - index * chunkBytes));
+                const slice = want === 0 ? buffer.subarray(0, 0) : await readExactly(body, buffer, want, bodyStart + index * chunkBytes);
+                const final = index === totalChunks - 1;
+                const nonce = chunkNonce(noncePrefix, index);
+                const cipher = aeadCipher(suite, archiveKey, nonce);
+                cipher.setAAD(chunkAad({ archiveId, suite, index, totalChunks, plaintextBytes: slice.length, final }));
+                const stored = Buffer.concat([cipher.update(slice), cipher.final(), cipher.getAuthTag()]);
+                bodyHash.update(slice);
+                if (!compressed) {
+                    originalHash.update(slice);
+                    originalBytes += slice.length;
+                    options.onProgress?.(originalBytes);
+                }
+                ciphertextHash.update(stored);
+                // A short write is not an error, so a disk filling up mid-chunk can
+                // land fewer bytes than were hashed. That would be caught eventually —
+                // the recorded ciphertext hash would not match the file — but only on
+                // the restore, which is far too late if the source has been reclaimed
+                // in between. Refusing here keeps the archive and the hash the same
+                // object for the same reason the destination path checks its errno.
+                const { bytesWritten } = await dest.write(stored);
+                if (bytesWritten !== stored.length) {
+                    throw new CryptoError("destination_write_failed", `Only ${bytesWritten} of ${stored.length} bytes of chunk ${index} reached ${destPath} — the volume is probably full`);
+                }
+                storedBytes += stored.length;
+                chunks.push({ index, plaintextBytes: slice.length, storedBytes: stored.length, sha256: sha256(stored), nonce: nonce.toString("base64"), final });
+            }
+        }
+        finally {
+            await body.close();
+            await dest.close();
+        }
+        const envelope = {
+            version: ENVELOPE_VERSION, archiveId, suite, chunkBytes, chunks,
+            manifest: {
+                plaintextSha256: bodyHash.digest("hex"), plaintextBytes: bodyBytes,
+                ...(compressed ? { compression: "gzip", originalBytes, originalSha256: originalHash.digest("hex") } : {}),
+                createdAt: options.createdAt ?? new Date().toISOString(), adapter: options.adapter
+            },
+            wrappedKeys: wrapAll(archiveKey, options.recipients, suite, archiveId, options.deterministic)
+        };
+        return { envelope, ciphertextSha256: ciphertextHash.digest("hex"), storedBytes, originalBytes, originalSha256: compressed ? envelope.manifest.originalSha256 : envelope.manifest.plaintextSha256 };
+    }
+    catch (error) {
+        await rm(destPath, { force: true });
+        throw error;
+    }
+    finally {
+        if (compressed)
+            await rm(bodyPath, { force: true });
+        if (!options.deterministic)
+            zeroize(archiveKey);
+    }
+}
+/**
+ * True for an errno the kernel handed back, as opposed to a cryptographic
+ * failure. Node stamps these as `ENOSPC`, `EACCES`, `EIO` and friends; zlib and
+ * this package's own failures never look like that.
+ */
+function isSystemError(error) {
+    const code = error?.code;
+    return error instanceof Error && typeof code === "string" && /^E[A-Z]+$/.test(code);
+}
+/**
+ * Reports a failure to write the *destination* as exactly that.
+ *
+ * Restoring writes decrypted bytes to a file, so it can fail for reasons that
+ * have nothing to do with the archive: a full disk, a read-only volume, a
+ * revoked permission. Those used to fall through to the decompression branch
+ * above and surface as a hash failure, which the vault layer then reported as a
+ * wrong recovery phrase — telling a user whose disk had filled that their
+ * archive was unreadable, when the archive was fine and the disk was not.
+ */
+function writeFailure(error, destPath) {
+    // Named by directory, not by file: callers restore through a temporary name
+    // they rename into place, and quoting that back at a user would be noise.
+    // The volume is what ran out of room anyway.
+    const where = dirname(destPath);
+    return new CryptoError("destination_write_failed", error.code === "ENOSPC"
+        ? `Ran out of space writing to ${where}. The archive is intact — free some space and restore again.`
+        : `Could not write to ${where} (${error.code}). The archive is intact.`);
+}
+/**
+ * Opens an archive straight to a file, holding one chunk at a time.
+ *
+ * Every chunk is authenticated before its bytes reach the destination, so what
+ * is written is never unauthenticated data — but the *whole-archive* manifest
+ * hashes can only be checked once the last chunk has gone past. A caller must
+ * therefore write to a temporary path and publish it only after this resolves;
+ * a rejection can leave a partial file behind, which is why `restoreArchive`
+ * renames into place rather than writing the destination directly.
+ */
+export async function openArchiveToFile(envelope, ciphertextPath, destPath, unlock) {
+    const storedOnDisk = (await stat(ciphertextPath)).size;
+    validateEnvelope(envelope, storedOnDisk);
+    const compression = envelope.manifest.compression;
+    if (compression !== undefined && compression !== "gzip" && compression !== "gzip-chunk")
+        throw new CryptoError("envelope_malformed", "Unsupported compression");
+    const archiveKey = unwrapArchiveKey(envelope.wrappedKeys, envelope.suite, envelope.archiveId, unlock);
+    if (archiveKey.length !== KEY_BYTES) {
+        zeroize(archiveKey);
+        throw new CryptoError("key_malformed", "Unwrapped archive key has the wrong length");
+    }
+    const handle = await open(ciphertextPath, "r");
+    const bodyHash = createHash("sha256");
+    const originalHash = createHash("sha256");
+    let bodyBytes = 0;
+    let originalBytes = 0;
+    let bodyDigest = "";
+    async function* authenticatedBody() {
+        const widest = envelope.chunks.reduce((most, chunk) => Math.max(most, chunk.storedBytes), 1);
+        const buffer = Buffer.allocUnsafe(widest);
+        let offset = 0;
+        for (const chunk of envelope.chunks) {
+            const stored = await readExactly(handle, buffer, chunk.storedBytes, offset);
+            offset += chunk.storedBytes;
+            if (sha256(stored) !== chunk.sha256)
+                throw new CryptoError("chunk_hash_mismatch", `Chunk ${chunk.index} does not match its recorded hash`);
+            const decipher = aeadDecipher(envelope.suite, archiveKey, Buffer.from(chunk.nonce, "base64"));
+            decipher.setAAD(chunkAad({ archiveId: envelope.archiveId, suite: envelope.suite, index: chunk.index, totalChunks: envelope.chunks.length, plaintextBytes: chunk.plaintextBytes, final: chunk.final }));
+            decipher.setAuthTag(stored.subarray(stored.length - TAG_BYTES));
+            let plain;
+            try {
+                plain = Buffer.concat([decipher.update(stored.subarray(0, stored.length - TAG_BYTES)), decipher.final()]);
+            }
+            catch {
+                throw new CryptoError("chunk_auth_failed", `Chunk ${chunk.index} failed authentication`);
+            }
+            bodyHash.update(plain);
+            bodyBytes += plain.length;
+            yield plain;
+        }
+        if (bodyBytes !== envelope.manifest.plaintextBytes)
+            throw new CryptoError("plaintext_length_mismatch", "Decrypted length does not match the manifest");
+        bodyDigest = bodyHash.digest("hex");
+        if (bodyDigest !== envelope.manifest.plaintextSha256)
+            throw new CryptoError("plaintext_hash_mismatch", "Decrypted bytes do not match the manifest hash");
+    }
+    async function* measure(chunks) {
+        for await (const piece of chunks) {
+            originalHash.update(piece);
+            originalBytes += piece.length;
+            yield piece;
+        }
+    }
+    try {
+        if (compression === "gzip-chunk") {
+            // Each authenticated piece is exactly one chunk's compressed slice —
+            // undone on its own, in chunk order. This is the same access pattern a
+            // ranged fetch will use, just walked start to end.
+            async function* perChunk(chunks) {
+                for await (const piece of chunks) {
+                    let slice;
+                    try {
+                        slice = gunzipSync(piece);
+                    }
+                    catch {
+                        throw new CryptoError("plaintext_hash_mismatch", "Decompression failed after authentication");
+                    }
+                    originalHash.update(slice);
+                    originalBytes += slice.length;
+                    yield slice;
+                }
+            }
+            try {
+                await pipeline(authenticatedBody(), perChunk, createWriteStream(destPath, { mode: 0o600 }));
+            }
+            catch (error) {
+                if (error instanceof CryptoError)
+                    throw error;
+                if (isSystemError(error))
+                    throw writeFailure(error, destPath);
+                throw error;
+            }
+            if (originalBytes !== envelope.manifest.originalBytes)
+                throw new CryptoError("plaintext_length_mismatch", "Decompressed length does not match the manifest");
+            const digest = originalHash.digest("hex");
+            if (digest !== envelope.manifest.originalSha256)
+                throw new CryptoError("plaintext_hash_mismatch", "Decompressed bytes do not match the manifest hash");
+            return { bytes: originalBytes, sha256: digest };
+        }
+        if (compression === "gzip") {
+            try {
+                await pipeline(authenticatedBody(), createGunzip(), measure, createWriteStream(destPath, { mode: 0o600 }));
+            }
+            catch (error) {
+                // A zlib failure here is not a tampering signal — every byte fed to the
+                // decompressor was authenticated first — but it still means the archive
+                // cannot produce the caller's bytes, so it is reported as such.
+                if (error instanceof CryptoError)
+                    throw error;
+                if (isSystemError(error))
+                    throw writeFailure(error, destPath);
+                throw new CryptoError("plaintext_hash_mismatch", "Decompression failed after authentication");
+            }
+            // The archive is honoured only once the caller's own bytes are proven, not
+            // merely the compressed stream they were derived from.
+            if (originalBytes !== envelope.manifest.originalBytes)
+                throw new CryptoError("plaintext_length_mismatch", "Decompressed length does not match the manifest");
+            const digest = originalHash.digest("hex");
+            if (digest !== envelope.manifest.originalSha256)
+                throw new CryptoError("plaintext_hash_mismatch", "Decompressed bytes do not match the manifest hash");
+            return { bytes: originalBytes, sha256: digest };
+        }
+        try {
+            await pipeline(authenticatedBody(), createWriteStream(destPath, { mode: 0o600 }));
+        }
+        catch (error) {
+            if (error instanceof CryptoError)
+                throw error;
+            if (isSystemError(error))
+                throw writeFailure(error, destPath);
+            throw error;
+        }
+        return { bytes: bodyBytes, sha256: bodyDigest };
+    }
+    finally {
+        await handle.close();
+        zeroize(archiveKey);
+    }
+}
+/**
+ * SHA-256 of several prefixes of a file, in one read and in constant memory.
+ *
+ * This is how an append is *proved* rather than assumed: hashing `[0, N)` of the
+ * live file and comparing it with what an archive recorded is the only evidence
+ * that the archive's bytes are still a prefix of it. Several offsets are
+ * answered together because a transcript usually carries several archives, and
+ * asking one at a time would re-read it once per archive.
+ *
+ * An offset past the end of the file is simply absent from the result.
+ */
+export async function hashFilePrefixes(path, offsets) {
+    const wanted = [...new Set(offsets)].filter((offset) => offset > 0).sort((a, b) => a - b);
+    const digests = new Map();
+    if (wanted.length === 0)
+        return digests;
+    // One pass, with the running digest snapshotted at each boundary: a transcript
+    // carrying a dozen snapshots costs one read of it, not a dozen.
+    const hash = createHash("sha256");
+    const limit = wanted[wanted.length - 1];
+    let read = 0;
+    let next = 0;
+    for await (const piece of createReadStream(path, { start: 0, end: limit - 1 })) {
+        const buffer = piece;
+        let offset = 0;
+        while (offset < buffer.length && next < wanted.length) {
+            const take = Math.min(buffer.length - offset, wanted[next] - read);
+            hash.update(buffer.subarray(offset, offset + take));
+            read += take;
+            offset += take;
+            while (next < wanted.length && wanted[next] === read)
+                digests.set(wanted[next++], hash.copy().digest("hex"));
+        }
+    }
+    return digests;
+}
+/**
+ * SHA-256 of a byte range of a file, read in constant memory.
+ *
+ * This is how an append is *proved* rather than assumed: hashing `[0, N)` of the
+ * live file and comparing it with what an archive recorded is the only evidence
+ * that the archive's bytes are still a prefix of it.
+ */
+export async function hashFileRange(path, start = 0, end) {
+    const size = (await stat(path)).size;
+    const last = Math.min(end ?? size, size);
+    const hash = createHash("sha256");
+    let bytes = 0;
+    if (last > start) {
+        for await (const piece of createReadStream(path, { start, end: last - 1 })) {
+            hash.update(piece);
+            bytes += piece.length;
+        }
+    }
+    return { sha256: hash.digest("hex"), bytes };
+}
+/**
+ * The single-pass gzip-chunk seal, destination-agnostic. This is what makes
+ * seal-to-cloud a sink implementation instead of an architecture project: the
+ * chunk loop neither knows nor cares whether `write` lands on a disk, a GCS
+ * resumable session, or an S3 multipart part buffer. Local cost is one slice
+ * in memory, full stop.
+ */
+export async function sealChunksToSink(source, sink, options) {
+    const suite = options.suite ?? "chacha20-poly1305";
+    const chunkBytes = options.chunkBytes ?? DEFAULT_CHUNK_BYTES;
+    const archiveId = options.archiveId ?? randomUUID();
+    const ownKey = !options.archiveKey;
+    const archiveKey = options.archiveKey ?? randomBytes(KEY_BYTES);
+    const noncePrefix = options.noncePrefix ?? randomBytes(4);
+    const firstChunkIndex = options.firstChunkIndex ?? 0;
+    if (!Number.isInteger(firstChunkIndex) || firstChunkIndex < 0)
+        throw new CryptoError("invalid_range", "firstChunkIndex must be a non-negative integer");
+    const start = source.start ?? 0;
+    const sourceBytes = (await stat(source.path)).size;
+    const end = Math.min(source.end ?? sourceBytes, sourceBytes);
+    if (start < 0 || end < start)
+        throw new CryptoError("invalid_range", `Range ${start}..${end} is not a range of ${source.path}`);
+    const raw = end - start;
+    const localChunks = Math.max(1, Math.ceil(raw / chunkBytes));
+    const totalChunks = firstChunkIndex + localChunks;
+    const chunks = [];
+    const originalHash = options.hashers?.plaintext ?? createHash("sha256");
+    const bodyHash = createHash("sha256");
+    const ciphertextHash = options.hashers?.stored ?? createHash("sha256");
+    let originalBytes = 0;
+    let bodyBytes = 0;
+    let storedBytesTotal = 0;
+    const buffer = Buffer.allocUnsafe(Math.min(chunkBytes, Math.max(raw, 1)));
+    const src = await open(source.path, "r");
+    try {
+        for (let position = 0; position < localChunks; position += 1) {
+            const index = firstChunkIndex + position;
+            const want = Math.max(0, Math.min(chunkBytes, raw - position * chunkBytes));
+            const slice = want === 0 ? buffer.subarray(0, 0) : await readExactly(src, buffer, want, start + position * chunkBytes);
+            originalHash.update(slice);
+            originalBytes += slice.length;
+            options.onProgress?.(originalBytes);
+            options.onPlaintext?.(index, slice);
+            const packed = gzipSync(slice, { level: 6 });
+            const final = index === totalChunks - 1;
+            const nonce = chunkNonce(noncePrefix, index);
+            const cipher = aeadCipher(suite, archiveKey, nonce);
+            cipher.setAAD(chunkAad({ archiveId, suite, index, totalChunks, plaintextBytes: packed.length, final }));
+            const stored = Buffer.concat([cipher.update(packed), cipher.final(), cipher.getAuthTag()]);
+            bodyHash.update(packed);
+            bodyBytes += packed.length;
+            ciphertextHash.update(stored);
+            await sink.write(stored);
+            storedBytesTotal += stored.length;
+            chunks.push({ index, plaintextBytes: packed.length, rawBytes: slice.length, storedBytes: stored.length, sha256: createHash("sha256").update(stored).digest("hex"), nonce: nonce.toString("base64"), final });
+        }
+    }
+    finally {
+        await src.close();
+    }
+    const originalSha256 = digestHex(originalHash);
+    const envelope = {
+        version: ENVELOPE_VERSION, archiveId, suite, chunkBytes, chunks,
+        manifest: {
+            plaintextSha256: bodyHash.digest("hex"), plaintextBytes: bodyBytes,
+            compression: "gzip-chunk", originalBytes, originalSha256,
+            createdAt: options.createdAt ?? new Date().toISOString(), adapter: options.adapter
+        },
+        wrappedKeys: wrapAll(archiveKey, options.recipients, suite, archiveId, options.deterministic)
+    };
+    if (ownKey && !options.deterministic)
+        zeroize(archiveKey);
+    return { envelope, ciphertextSha256: digestHex(ciphertextHash), storedBytes: storedBytesTotal, originalBytes, originalSha256 };
+}