sealkeep 0.9.0 → 0.11.0

package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/envelope.js

@@ -3,6 +3,7 @@ import { createHash, randomBytes, randomUUID } from "node:crypto";
 import { aeadCipher, aeadDecipher } from "./aead.js";
 import { CryptoError, unwrapArchiveKey, wrapForKey, wrapForPhrase, x25519PublicKeyFromRaw, zeroize } from "./recipients.js";
 import { DEFAULT_CHUNK_BYTES, ENVELOPE_VERSION, KEY_BYTES, NONCE_BYTES, SUITES, TAG_BYTES, chunkAad, chunkNonce } from "./format.js";
+import { squeezeSync, unsqueezeSync } from "./squeeze.js";
 const sha256 = (input) => createHash("sha256").update(input).digest("hex");
 function toPublicKey(value) {
     return Buffer.isBuffer(value) ? x25519PublicKeyFromRaw(value) : value;
@@ -38,9 +39,10 @@ export function encryptArchive(plaintext, options) {
     const noncePrefix = options.deterministic?.noncePrefix ?? randomBytes(4);
     if (archiveKey.length !== KEY_BYTES)
         throw new CryptoError("key_malformed", `An archive key must be ${KEY_BYTES} bytes`);
-    if ((options.compression ?? "none") === "gzip-chunk")
-        throw new CryptoError("envelope_malformed", "gzip-chunk sealing is streaming-only; use sealArchiveToFile");
-    const body = (options.compression ?? "none") === "gzip" ? gzipSync(plaintext, { level: 6 }) : plaintext;
+    if (/-chunk$/.test(options.compression ?? "none"))
+        throw new CryptoError("envelope_malformed", `${options.compression} sealing is streaming-only; use sealArchiveToFile`);
+    const codecIn = options.compression ?? "none";
+    const body = codecIn === "gzip" ? gzipSync(plaintext, { level: 6 }) : codecIn === "zstd" ? squeezeSync(plaintext) : plaintext;
     const totalChunks = Math.max(1, Math.ceil(body.length / chunkBytes));
     const chunks = [];
     const blocks = [];
@@ -58,8 +60,8 @@ export function encryptArchive(plaintext, options) {
         const envelope = {
             version: ENVELOPE_VERSION, archiveId, suite, chunkBytes, chunks,
             manifest: { plaintextSha256: sha256(body), plaintextBytes: body.length,
-                ...((options.compression ?? "none") === "gzip"
-                    ? { compression: "gzip", originalBytes: plaintext.length, originalSha256: sha256(plaintext) }
+                ...(codecIn === "gzip" || codecIn === "zstd"
+                    ? { compression: codecIn, originalBytes: plaintext.length, originalSha256: sha256(plaintext) }
                     : {}), createdAt: options.createdAt ?? new Date().toISOString(), adapter: options.adapter },
             wrappedKeys: wrapAll(archiveKey, options.recipients, suite, archiveId, options.deterministic)
         };
@@ -100,20 +102,25 @@ export function validateEnvelope(envelope, ciphertextBytes) {
         throw new CryptoError("envelope_malformed", "Chunk lengths do not sum to the manifest plaintext length");
 }
 /**
- * Decrypts an archive after verifying its structure and every stored chunk hash.
- * Every chunk is authenticated against associated data that pins its position and
- * the total chunk count, so a reordered or truncated archive fails rather than
- * returning partial plaintext.
+ * Authenticates and decrypts the stored body without expanding its compression.
+ *
+ * Keeping the authenticated chunks separate matters for callers that can stream
+ * decompression. A highly-compressible search index may be tens of megabytes on
+ * disk and more than a gigabyte after gzip; forcing that expanded body through a
+ * single Buffer is enough to OOM a long-lived agent process before it can inspect
+ * one query term.
  */
-export function decryptArchive(envelope, ciphertext, unlock) {
+export function decryptArchiveBodyChunks(envelope, ciphertext, unlock) {
     validateEnvelope(envelope, ciphertext.length);
     const archiveKey = unwrapArchiveKey(envelope.wrappedKeys, envelope.suite, envelope.archiveId, unlock);
     if (archiveKey.length !== KEY_BYTES) {
         zeroize(archiveKey);
         throw new CryptoError("key_malformed", "Unwrapped archive key has the wrong length");
     }
+    const parts = [];
     try {
-        const parts = [];
+        const bodyHash = createHash("sha256");
+        let bodyBytes = 0;
         let offset = 0;
         for (const chunk of envelope.chunks) {
             const stored = ciphertext.subarray(offset, offset + chunk.storedBytes);
@@ -124,30 +131,57 @@ export function decryptArchive(envelope, ciphertext, unlock) {
             decipher.setAAD(chunkAad({ archiveId: envelope.archiveId, suite: envelope.suite, index: chunk.index, totalChunks: envelope.chunks.length, plaintextBytes: chunk.plaintextBytes, final: chunk.final }));
             decipher.setAuthTag(stored.subarray(stored.length - TAG_BYTES));
             try {
-                parts.push(Buffer.concat([decipher.update(stored.subarray(0, stored.length - TAG_BYTES)), decipher.final()]));
+                const part = Buffer.concat([decipher.update(stored.subarray(0, stored.length - TAG_BYTES)), decipher.final()]);
+                parts.push(part);
+                bodyHash.update(part);
+                bodyBytes += part.length;
             }
             catch {
                 throw new CryptoError("chunk_auth_failed", `Chunk ${chunk.index} failed authentication`);
             }
         }
-        const plaintext = Buffer.concat(parts);
-        if (plaintext.length !== envelope.manifest.plaintextBytes)
+        if (bodyBytes !== envelope.manifest.plaintextBytes)
             throw new CryptoError("plaintext_length_mismatch", "Decrypted length does not match the manifest");
-        if (sha256(plaintext) !== envelope.manifest.plaintextSha256)
+        if (bodyHash.digest("hex") !== envelope.manifest.plaintextSha256)
             throw new CryptoError("plaintext_hash_mismatch", "Decrypted bytes do not match the manifest hash");
+        return parts;
+    }
+    catch (error) {
+        for (const part of parts)
+            zeroize(part);
+        throw error;
+    }
+    finally {
+        zeroize(archiveKey);
+    }
+}
+/**
+ * Decrypts an archive after verifying its structure and every stored chunk hash.
+ * Every chunk is authenticated against associated data that pins its position and
+ * the total chunk count, so a reordered or truncated archive fails rather than
+ * returning partial plaintext.
+ */
+export function decryptArchive(envelope, ciphertext, unlock) {
+    const parts = decryptArchiveBodyChunks(envelope, ciphertext, unlock);
+    const plaintext = Buffer.concat(parts);
+    try {
         // Absent on archives written without compression, where the authenticated
         // body already is the caller's plaintext.
         if (envelope.manifest.compression === undefined)
             return plaintext;
-        if (envelope.manifest.compression !== "gzip" && envelope.manifest.compression !== "gzip-chunk")
+        const codec = envelope.manifest.compression;
+        if (codec !== "gzip" && codec !== "gzip-chunk" && codec !== "zstd" && codec !== "zstd-chunk")
             throw new CryptoError("envelope_malformed", "Unsupported compression");
+        // Which compressor sealed this archive is recorded in its own envelope,
+        // so gzip archives written before zstd existed keep opening unchanged.
+        const expand = codec === "zstd" || codec === "zstd-chunk" ? unsqueezeSync : gunzipSync;
         let original;
         try {
-            original = envelope.manifest.compression === "gzip-chunk"
+            original = codec.endsWith("-chunk")
                 // Each part is one chunk's compressed slice, independently made and
                 // independently undone — concatenating the results reproduces the file.
-                ? Buffer.concat(parts.map((part) => gunzipSync(part)))
-                : gunzipSync(plaintext);
+                ? Buffer.concat(parts.map((part) => expand(part)))
+                : expand(plaintext);
         }
         catch {
             throw new CryptoError("plaintext_hash_mismatch", "Decompression failed after authentication");
@@ -160,8 +194,11 @@ export function decryptArchive(envelope, ciphertext, unlock) {
             throw new CryptoError("plaintext_hash_mismatch", "Decompressed bytes do not match the manifest hash");
         return original;
     }
-    finally {
-        zeroize(archiveKey);
+    catch (error) {
+        zeroize(plaintext);
+        for (const part of parts)
+            zeroize(part);
+        throw error;
     }
 }
 /**

package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/format.d.ts

@@ -45,7 +45,7 @@ export type ChunkHeader = {
     index: number;
     /** Bytes this chunk's ciphertext decrypts to. Under "gzip-chunk" that is the compressed slice; the raw length lives in `rawBytes`. */
     plaintextBytes: number;
-    /** Original (pre-compression) bytes this chunk covers. Present only under "gzip-chunk", where it is what makes byte-ranged retrieval computable without decompressing anything. */
+    /** Original (pre-compression) bytes this chunk covers. Present under either chunked codec, where it is what makes byte-ranged retrieval computable without decompressing anything. */
     rawBytes?: number;
     /** Stored bytes for this chunk: ciphertext followed by its authentication tag. */
     storedBytes: number;
@@ -66,7 +66,7 @@ export type Envelope = {
          * every chunk decrypts AND decompresses on its own — what makes streaming
          * seals and fetch-only-the-chunks retrieval possible. Absent means none.
          */
-        compression?: "gzip" | "gzip-chunk";
+        compression?: "gzip" | "gzip-chunk" | "zstd" | "zstd-chunk";
         originalBytes?: number;
         originalSha256?: string;
         plaintextSha256: string;

package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/index.d.ts

@@ -3,3 +3,4 @@ export * from "./recipients.js";
 export * from "./envelope.js";
 export * from "./stream.js";
 export * from "./sha256-stream.js";
+export * from "./squeeze.js";

package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/index.js

@@ -3,3 +3,4 @@ export * from "./recipients.js";
 export * from "./envelope.js";
 export * from "./stream.js";
 export * from "./sha256-stream.js";
+export * from "./squeeze.js";

package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/recipients.js

@@ -38,10 +38,10 @@ function openKey(suite, kek, nonce, sealed, aad) {
     }
 }
 export function phraseRecipientId(salt) {
-    return createHash("sha256").update(Buffer.concat([Buffer.from("vaultline-phrase-recipient:v2"), salt])).digest("hex").slice(0, 32);
+    return createHash("sha256").update(Buffer.concat([Buffer.from("sealkeep-phrase-recipient:v2"), salt])).digest("hex").slice(0, 32);
 }
 export function keyRecipientId(publicKeyRaw) {
-    return createHash("sha256").update(Buffer.concat([Buffer.from("vaultline-key-recipient:v2"), publicKeyRaw])).digest("hex").slice(0, 32);
+    return createHash("sha256").update(Buffer.concat([Buffer.from("sealkeep-key-recipient:v2"), publicKeyRaw])).digest("hex").slice(0, 32);
 }
 export function rawPublicKey(key) {
     // The last 32 bytes of an X25519 SPKI DER encoding are the raw public key.
@@ -67,7 +67,7 @@ export function wrapForKey(archiveKey, recipientPublicKey, suite, archiveId, opt
     const recipientRaw = rawPublicKey(recipientPublicKey);
     const ephemeralRaw = rawPublicKey(ephemeral.publicKey);
     const shared = diffieHellman({ privateKey: ephemeral.privateKey, publicKey: recipientPublicKey });
-    const kek = Buffer.from(hkdfSync("sha256", shared, Buffer.concat([recipientRaw, ephemeralRaw]), Buffer.from("vaultline-recipient:v2"), KEY_BYTES));
+    const kek = Buffer.from(hkdfSync("sha256", shared, Buffer.concat([recipientRaw, ephemeralRaw]), Buffer.from("sealkeep-recipient:v2"), KEY_BYTES));
     const id = keyRecipientId(recipientRaw);
     try {
         return { type: "x25519", id, recipientPublicKey: b64(recipientRaw), ephemeralPublicKey: b64(ephemeralRaw), nonce: b64(options.nonce), ciphertext: b64(sealKey(suite, kek, options.nonce, archiveKey, wrapAad(archiveId, id))) };
@@ -112,7 +112,7 @@ export function unwrapArchiveKey(wrappedKeys, suite, archiveId, unlock) {
         if ("privateKey" in unlock && wrapped.type === "x25519") {
             const privateKey = typeof unlock.privateKey === "string" ? createPrivateKey(unlock.privateKey) : unlock.privateKey;
             const shared = diffieHellman({ privateKey, publicKey: x25519PublicKeyFromRaw(unb64(wrapped.ephemeralPublicKey)) });
-            const kek = Buffer.from(hkdfSync("sha256", shared, Buffer.concat([unb64(wrapped.recipientPublicKey), unb64(wrapped.ephemeralPublicKey)]), Buffer.from("vaultline-recipient:v2"), KEY_BYTES));
+            const kek = Buffer.from(hkdfSync("sha256", shared, Buffer.concat([unb64(wrapped.recipientPublicKey), unb64(wrapped.ephemeralPublicKey)]), Buffer.from("sealkeep-recipient:v2"), KEY_BYTES));
             try {
                 return openKey(suite, kek, unb64(wrapped.nonce), unb64(wrapped.ciphertext), wrapAad(archiveId, wrapped.id));
             }

package/dist/packages/sealkeep-crypto/src/squeeze.d.ts

@@ -0,0 +1,6 @@
+export declare const squeezeStream: () => import("zlib").ZstdCompress;
+export declare const unsqueezeStream: () => import("zlib").ZstdDecompress;
+export declare const squeezeSync: (input: Buffer) => NonSharedBuffer;
+export declare const unsqueezeSync: (input: Buffer, options?: {
+    maxOutputLength?: number;
+}) => NonSharedBuffer;

package/dist/packages/sealkeep-crypto/src/squeeze.js

@@ -0,0 +1,39 @@
+import { constants as Z, createZstdCompress, createZstdDecompress, zstdCompressSync, zstdDecompressSync } from "node:zlib";
+/**
+ * The compressor, and why it is not gzip.
+ *
+ * An agent transcript is not ordinary text. Every time the agent compacts its
+ * context it writes the whole conversation back to disk again — screenshots
+ * included — so one session file can hold the same 300 KB PNG fifty-odd times.
+ * Measured on a real 1.9 GB Codex session: 28 distinct images, each stored an
+ * average of 56 times, 90% of the file.
+ *
+ * gzip cannot see any of that. Its window is 32 KB, so a duplicate 300 KB image
+ * a hundred megabytes upstream is invisible and gets stored again in full; on
+ * that file it managed 1.5x. zstd with long-distance matching keeps a 128 MB
+ * window and collapses the repeats: the same file goes to 36 MB — 54x — and it
+ * does it faster than the gzip it replaces, because most of the input turns
+ * into a back-reference instead of being scanned.
+ *
+ * WINDOW_LOG is the whole trick and the one number worth guarding. At 4 MB
+ * chunks the same compressor gets 2x, because nothing repeats inside a 4 MB
+ * window; at whole-file scope it gets 59x. Compression here has to see the
+ * whole file, or it is not worth changing.
+ *
+ * Built into node:zlib since Node 22.15, so this costs no dependency and no
+ * new licence. Nothing decides to *read* zstd from these helpers: the archive's
+ * own envelope records which compressor sealed it, and gzip archives written
+ * before this existed keep opening through the gzip path, forever.
+ */
+const WINDOW_LOG = 27;
+const COMPRESS = {
+    [Z.ZSTD_c_compressionLevel]: 9,
+    [Z.ZSTD_c_enableLongDistanceMatching]: 1,
+    [Z.ZSTD_c_windowLog]: WINDOW_LOG,
+};
+/** Readers must be told to allocate the same window the writer used. */
+const DECOMPRESS = { [Z.ZSTD_d_windowLogMax]: WINDOW_LOG };
+export const squeezeStream = () => createZstdCompress({ params: { ...COMPRESS } });
+export const unsqueezeStream = () => createZstdDecompress({ params: { ...DECOMPRESS } });
+export const squeezeSync = (input) => zstdCompressSync(input, { params: { ...COMPRESS } });
+export const unsqueezeSync = (input, options = {}) => zstdDecompressSync(input, { ...options, params: { ...DECOMPRESS } });

package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/stream.d.ts

@@ -36,6 +36,20 @@ export type StreamEncryptOptions = EncryptOptions & {
     scratchDir?: string;
     /** Called as the source is read, with cumulative bytes so far — a person watching a 1.6 GB seal deserves a number that moves. Throttling is the caller's business. */
     onProgress?: (originalBytes: number) => void;
+    /** Cancels source reads, compression, sink writes and unpublished output. */
+    signal?: AbortSignal;
+    /**
+     * Cooperative scheduling checkpoint for low-priority callers. It is awaited
+     * after each bounded amount of compression/encryption work, so a streaming
+     * seal cannot monopolise a core merely because its memory use is bounded.
+     */
+    onYield?: () => void | Promise<void>;
+    /**
+     * Each chunk's plaintext, as it is sealed (chunk-compressed layouts only —
+     * a whole-file body has no chunks to report). The caller that indexes what
+     * it seals reads the source once, here, instead of decrypting it again.
+     */
+    onPlaintext?: (chunkIndex: number, slice: Buffer) => void;
 };
 export type SealResult = {
     envelope: Envelope;
@@ -58,6 +72,16 @@ export type OpenResult = {
     bytes: number;
     sha256: string;
 };
+export type StreamOpenOptions = {
+    /** Reject before allocating the reusable ciphertext buffer if any stored chunk exceeds this many bytes. */
+    maxStoredChunkBytes?: number;
+    /** Reject both a larger declared original and actual decompression output beyond this many bytes. */
+    maxOriginalBytes?: number;
+    /** Reject one independently-compressed chunk after this many expanded bytes. */
+    maxExpandedChunkBytes?: number;
+    /** Expand independently-compressed chunks through a stream instead of a whole-chunk sync allocation. */
+    streamChunkDecompression?: boolean;
+};
 /**
  * Opens an archive straight to a file, holding one chunk at a time.
  *
@@ -68,7 +92,16 @@ export type OpenResult = {
  * a rejection can leave a partial file behind, which is why `restoreArchive`
  * renames into place rather than writing the destination directly.
  */
-export declare function openArchiveToFile(envelope: Envelope, ciphertextPath: string, destPath: string, unlock: Unlock): Promise<OpenResult>;
+/**
+ * `destPath` may be a Writable instead of a path. Verification needs to prove an
+ * archive decrypts to its recorded bytes, and materialising the plaintext to do
+ * so costs as much free disk as the original — on the nearly-full disk that
+ * makes someone reach for this product in the first place, that is exactly the
+ * space they do not have, so the proof became impossible precisely when it
+ * mattered. Every integrity check below runs the same either way; only the
+ * destination changes.
+ */
+export declare function openArchiveToFile(envelope: Envelope, ciphertextPath: string, destPath: string | NodeJS.WritableStream, unlock: Unlock, options?: StreamOpenOptions): Promise<OpenResult>;
 /**
  * SHA-256 of several prefixes of a file, in one read and in constant memory.
  *
@@ -80,7 +113,7 @@ export declare function openArchiveToFile(envelope: Envelope, ciphertextPath: st
  *
  * An offset past the end of the file is simply absent from the result.
  */
-export declare function hashFilePrefixes(path: string, offsets: readonly number[]): Promise<Map<number, string>>;
+export declare function hashFilePrefixes(path: string, offsets: readonly number[], onProgress?: (bytesRead: number) => void, signal?: AbortSignal, onYield?: () => void | Promise<void>): Promise<Map<number, string>>;
 /**
  * SHA-256 of a byte range of a file, read in constant memory.
  *
@@ -88,7 +121,7 @@ export declare function hashFilePrefixes(path: string, offsets: readonly number[
  * live file and comparing it with what an archive recorded is the only evidence
  * that the archive's bytes are still a prefix of it.
  */
-export declare function hashFileRange(path: string, start?: number, end?: number): Promise<{
+export declare function hashFileRange(path: string, start?: number, end?: number, onProgress?: (bytesRead: number) => void, signal?: AbortSignal, onYield?: () => void | Promise<void>): Promise<{
     sha256: string;
     bytes: number;
 }>;