sealkeep 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +163 -14
- package/CHANGELOG.md +223 -1
- package/CONTROL_PLANE.md +2 -2
- package/LICENSE +1 -1
- package/README.md +108 -22
- package/THIRD_PARTY.md +2 -2
- package/THREAT_MODEL.md +23 -4
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/chunk-access.d.ts +26 -4
- package/dist/packages/sealkeep-crypto/src/chunk-access.js +219 -0
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/envelope.d.ts +11 -1
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/envelope.js +58 -21
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/format.d.ts +2 -2
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/index.d.ts +1 -0
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/index.js +1 -0
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/recipients.js +4 -4
- package/dist/packages/sealkeep-crypto/src/squeeze.d.ts +6 -0
- package/dist/packages/sealkeep-crypto/src/squeeze.js +39 -0
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/stream.d.ts +36 -3
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/stream.js +196 -37
- package/dist/site/index.html +1808 -1904
- package/dist/site/llms.txt +67 -0
- package/dist/site/trust/architecture-data-flow.html +53 -0
- package/dist/site/trust/audit-roadmap.html +37 -0
- package/dist/site/trust/deployment-responsibility.html +11 -0
- package/dist/site/trust/dpa-sample.html +30 -0
- package/dist/site/trust/release-provenance.html +21 -0
- package/dist/site/trust/subprocessors.html +15 -0
- package/dist/site/trust/threat-model.html +34 -0
- package/dist/site/trust/toms.html +41 -0
- package/dist/site/trust-document.css +32 -0
- package/dist/site/trust.html +73 -0
- package/dist/site/visual/assets/index-8Nxnkc7o.js +207 -0
- package/dist/site/visual/assets/index-oxLy1bpy.css +1 -0
- package/dist/site/visual/index.html +18 -0
- package/dist/site.zip +0 -0
- package/dist/src/activity.d.ts +9 -0
- package/dist/src/activity.js +90 -1
- package/dist/src/adapters.d.ts +175 -5
- package/dist/src/adapters.js +961 -73
- package/dist/src/agent-context.d.ts +135 -0
- package/dist/src/agent-context.js +1059 -0
- package/dist/src/archive-copies.d.ts +47 -0
- package/dist/src/archive-copies.js +179 -0
- package/dist/src/audit.d.ts +1 -1
- package/dist/src/audit.js +29 -4
- package/dist/src/autopilot.d.ts +52 -7
- package/dist/src/autopilot.js +143 -25
- package/dist/src/background-bandwidth.d.ts +46 -0
- package/dist/src/background-bandwidth.js +301 -0
- package/dist/src/background-cpu.d.ts +82 -0
- package/dist/src/background-cpu.js +212 -0
- package/dist/src/background-worker-error.d.ts +12 -0
- package/dist/src/background-worker-error.js +18 -0
- package/dist/src/branding.d.ts +24 -1
- package/dist/src/branding.js +31 -1
- package/dist/src/bridge.d.ts +233 -0
- package/dist/src/bridge.js +604 -0
- package/dist/src/byte-stream.d.ts +91 -0
- package/dist/src/byte-stream.js +385 -0
- package/dist/src/chunk-store.d.ts +41 -8
- package/dist/src/chunk-store.js +161 -65
- package/dist/src/cli.js +1698 -163
- package/dist/src/cloud.d.ts +841 -31
- package/dist/src/cloud.js +3196 -277
- package/dist/src/context-background.d.ts +37 -0
- package/dist/src/context-background.js +309 -0
- package/dist/src/context-drain-child.d.ts +1 -0
- package/dist/src/context-drain-child.js +98 -0
- package/dist/src/context-reader.d.ts +118 -0
- package/dist/src/context-reader.js +447 -0
- package/dist/src/control-plane/auth.d.ts +32 -4
- package/dist/src/control-plane/auth.js +85 -24
- package/dist/src/control-plane/server.js +19 -6
- package/dist/src/control-plane.d.ts +17 -1
- package/dist/src/control-plane.js +32 -6
- package/dist/src/crypto.d.ts +1 -1
- package/dist/src/crypto.js +5 -5
- package/dist/src/daemon-lease.d.ts +70 -0
- package/dist/src/daemon-lease.js +420 -0
- package/dist/src/daemon.d.ts +94 -1
- package/dist/src/daemon.js +1082 -105
- package/dist/src/darwin-service-policy.d.ts +41 -0
- package/dist/src/darwin-service-policy.js +60 -0
- package/dist/src/dashboard-cli.js +15 -15
- package/dist/src/device-authorization.d.ts +37 -0
- package/dist/src/device-authorization.js +199 -0
- package/dist/src/device-enrollment.d.ts +91 -0
- package/dist/src/device-enrollment.js +349 -0
- package/dist/src/disk.d.ts +17 -12
- package/dist/src/disk.js +43 -17
- package/dist/src/doctor.d.ts +35 -1
- package/dist/src/doctor.js +316 -41
- package/dist/src/durable-ticket-lock.d.ts +24 -0
- package/dist/src/durable-ticket-lock.js +232 -0
- package/dist/src/enroll.d.ts +1 -1
- package/dist/src/enroll.js +13 -7
- package/dist/src/env.d.ts +10 -1
- package/dist/src/env.js +11 -3
- package/dist/src/errors.d.ts +8 -8
- package/dist/src/errors.js +6 -6
- package/dist/src/flush.d.ts +12 -0
- package/dist/src/flush.js +37 -0
- package/dist/src/heartbeat.d.ts +86 -12
- package/dist/src/heartbeat.js +415 -29
- package/dist/src/index-background-watchdog.d.ts +1 -0
- package/dist/src/index-background-watchdog.js +94 -0
- package/dist/src/index-background-work.d.ts +21 -0
- package/dist/src/index-background-work.js +25 -0
- package/dist/src/index-background.d.ts +64 -0
- package/dist/src/index-background.js +394 -0
- package/dist/src/index-build-child.d.ts +1 -0
- package/dist/src/index-build-child.js +109 -0
- package/dist/src/index-manifest.d.ts +52 -0
- package/dist/src/index-manifest.js +444 -0
- package/dist/src/index-publication-proof.d.ts +84 -0
- package/dist/src/index-publication-proof.js +380 -0
- package/dist/src/index-publication-state.d.ts +149 -0
- package/dist/src/index-publication-state.js +696 -0
- package/dist/src/index-publication-verifier.d.ts +89 -0
- package/dist/src/index-publication-verifier.js +341 -0
- package/dist/src/index-publish.d.ts +62 -0
- package/dist/src/index-publish.js +540 -0
- package/dist/src/index-scratch-cleanup.d.ts +19 -0
- package/dist/src/index-scratch-cleanup.js +166 -0
- package/dist/src/index-segment-types.d.ts +132 -0
- package/dist/src/index-segment-types.js +21 -0
- package/dist/src/index-segments.d.ts +9 -0
- package/dist/src/index-segments.js +516 -0
- package/dist/src/index-store.d.ts +123 -0
- package/dist/src/index-store.js +495 -0
- package/dist/src/index-sync.d.ts +91 -15
- package/dist/src/index-sync.js +286 -53
- package/dist/src/index-upgrade-publication.d.ts +30 -0
- package/dist/src/index-upgrade-publication.js +179 -0
- package/dist/src/integration-manager.d.ts +32 -0
- package/dist/src/integration-manager.js +394 -0
- package/dist/src/leakscan.js +1 -1
- package/dist/src/local-api.d.ts +56 -15
- package/dist/src/local-api.js +4979 -541
- package/dist/src/machine-settings.d.ts +51 -0
- package/dist/src/machine-settings.js +166 -0
- package/dist/src/managed-chunks.d.ts +5 -2
- package/dist/src/managed-chunks.js +14 -14
- package/dist/src/mcp-install.d.ts +11 -9
- package/dist/src/mcp-install.js +73 -28
- package/dist/src/mcp-workspace.d.ts +18 -0
- package/dist/src/mcp-workspace.js +50 -0
- package/dist/src/mcp.js +294 -25
- package/dist/src/migrate.js +27 -21
- package/dist/src/notify.d.ts +1 -1
- package/dist/src/notify.js +5 -5
- package/dist/src/offload.d.ts +201 -14
- package/dist/src/offload.js +1848 -140
- package/dist/src/onboarding.d.ts +8 -1
- package/dist/src/onboarding.js +4 -4
- package/dist/src/packages.d.ts +2 -2
- package/dist/src/packages.js +10 -2
- package/dist/src/passkey.d.ts +0 -1
- package/dist/src/passkey.js +2 -7
- package/dist/src/password-lock.d.ts +2 -2
- package/dist/src/password-lock.js +6 -6
- package/dist/src/paths.d.ts +2 -0
- package/dist/src/paths.js +2 -0
- package/dist/src/presence.d.ts +86 -0
- package/dist/src/presence.js +240 -0
- package/dist/src/progress-deadline.d.ts +21 -0
- package/dist/src/progress-deadline.js +91 -0
- package/dist/src/project-repair.d.ts +55 -0
- package/dist/src/project-repair.js +131 -0
- package/dist/src/providers/gcs.d.ts +28 -7
- package/dist/src/providers/gcs.js +35 -24
- package/dist/src/providers/gdrive.d.ts +71 -8
- package/dist/src/providers/gdrive.js +223 -50
- package/dist/src/providers/index.d.ts +11 -3
- package/dist/src/providers/index.js +30 -10
- package/dist/src/providers/s3.d.ts +30 -8
- package/dist/src/providers/s3.js +41 -30
- package/dist/src/providers/safe-storage-fetch.d.ts +12 -0
- package/dist/src/providers/safe-storage-fetch.js +72 -0
- package/dist/src/queue.d.ts +136 -19
- package/dist/src/queue.js +862 -96
- package/dist/src/reclaim-transaction.d.ts +156 -0
- package/dist/src/reclaim-transaction.js +1027 -0
- package/dist/src/recovery-codes.d.ts +32 -0
- package/dist/src/recovery-codes.js +338 -0
- package/dist/src/recovery.js +12 -9
- package/dist/src/rehydrate.d.ts +25 -22
- package/dist/src/rehydrate.js +319 -23
- package/dist/src/restore.d.ts +57 -4
- package/dist/src/restore.js +272 -36
- package/dist/src/resume-stub.d.ts +92 -0
- package/dist/src/resume-stub.js +417 -0
- package/dist/src/retention.d.ts +98 -7
- package/dist/src/retention.js +1066 -62
- package/dist/src/rotate.js +3 -3
- package/dist/src/search.d.ts +566 -8
- package/dist/src/search.js +5897 -290
- package/dist/src/secrets.d.ts +51 -7
- package/dist/src/secrets.js +316 -24
- package/dist/src/service.d.ts +49 -11
- package/dist/src/service.js +776 -35
- package/dist/src/share.js +3 -3
- package/dist/src/shared-spaces.d.ts +98 -0
- package/dist/src/shared-spaces.js +214 -0
- package/dist/src/source-reader.d.ts +73 -0
- package/dist/src/source-reader.js +715 -0
- package/dist/src/spool.d.ts +1 -1
- package/dist/src/spool.js +1 -1
- package/dist/src/start-tui.js +2 -1
- package/dist/src/start.js +2 -2
- package/dist/src/storage-endpoint.d.ts +21 -0
- package/dist/src/storage-endpoint.js +122 -0
- package/dist/src/storage-setup.js +12 -12
- package/dist/src/storage-targets.d.ts +109 -6
- package/dist/src/storage-targets.js +975 -67
- package/dist/src/stream-to-cloud.d.ts +5 -1
- package/dist/src/stream-to-cloud.js +34 -14
- package/dist/src/sync-rules.d.ts +31 -6
- package/dist/src/sync-rules.js +153 -14
- package/dist/src/team-backfill-scheduling.d.ts +8 -0
- package/dist/src/team-backfill-scheduling.js +33 -0
- package/dist/src/team-backfill.d.ts +116 -0
- package/dist/src/team-backfill.js +1429 -0
- package/dist/src/team-index-cache.d.ts +16 -0
- package/dist/src/team-index-cache.js +152 -0
- package/dist/src/team-offboarding.d.ts +38 -0
- package/dist/src/team-offboarding.js +1043 -0
- package/dist/src/team-presence.d.ts +127 -0
- package/dist/src/team-presence.js +904 -0
- package/dist/src/team-publication-policy.d.ts +20 -0
- package/dist/src/team-publication-policy.js +140 -0
- package/dist/src/team-realtime.d.ts +68 -0
- package/dist/src/team-realtime.js +816 -0
- package/dist/src/team-source-facts-cache.d.ts +23 -0
- package/dist/src/team-source-facts-cache.js +255 -0
- package/dist/src/trash.d.ts +1 -1
- package/dist/src/trash.js +2 -2
- package/dist/src/tui.js +11 -12
- package/dist/src/types.d.ts +173 -7
- package/dist/src/types.js +20 -0
- package/dist/src/ui-server.d.ts +163 -35
- package/dist/src/ui-server.js +712 -72
- package/dist/src/ui.d.ts +1 -2
- package/dist/src/ui.js +1 -2
- package/dist/src/upload.d.ts +27 -0
- package/dist/src/upload.js +383 -43
- package/dist/src/vault.d.ts +226 -30
- package/dist/src/vault.js +1776 -192
- package/dist/src/watcher.d.ts +7 -1
- package/dist/src/watcher.js +198 -55
- package/dist/src/worker.d.ts +27 -3
- package/dist/src/worker.js +274 -55
- package/package.json +33 -12
- package/scripts/native-reboot-rehearsal.mjs +90 -0
- package/web/app.js +6032 -343
- package/web/bootstrap.js +17 -0
- package/web/index.html +255 -57
- package/web/rail.js +317 -40
- package/web/retention.html +2 -2
- package/web/rules-view.js +188 -16
- package/web/sessions-view.js +485 -62
- package/web/sessions.html +2 -2
- package/web/setup-api.js +152 -29
- package/web/setup-logic.js +68 -9
- package/web/setup.html +113 -44
- package/web/setup.js +604 -71
- package/web/style.css +513 -98
- package/dist/packages/vaultline-crypto/src/chunk-access.js +0 -93
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/aead.d.ts +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/aead.js +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/format.js +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/recipients.d.ts +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/sha256-stream.d.ts +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/sha256-stream.js +0 -0
|
@@ -17,7 +17,9 @@
|
|
|
17
17
|
*/
|
|
18
18
|
import { createReadStream, createWriteStream } from "node:fs";
|
|
19
19
|
import { open, rm, stat } from "node:fs/promises";
|
|
20
|
+
import { Readable } from "node:stream";
|
|
20
21
|
import { pipeline } from "node:stream/promises";
|
|
22
|
+
import { setImmediate as yieldToEventLoop } from "node:timers/promises";
|
|
21
23
|
import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
|
|
22
24
|
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
23
25
|
import { dirname, join } from "node:path";
|
|
@@ -26,7 +28,9 @@ import { CryptoError, unwrapArchiveKey, zeroize } from "./recipients.js";
|
|
|
26
28
|
import { StreamingSha256 } from "./sha256-stream.js";
|
|
27
29
|
import { wrapAll, validateEnvelope } from "./envelope.js";
|
|
28
30
|
import { DEFAULT_CHUNK_BYTES, ENVELOPE_VERSION, KEY_BYTES, SUITES, TAG_BYTES, chunkAad, chunkNonce } from "./format.js";
|
|
31
|
+
import { squeezeStream, squeezeSync, unsqueezeStream, unsqueezeSync } from "./squeeze.js";
|
|
29
32
|
const sha256 = (input) => createHash("sha256").update(input).digest("hex");
|
|
33
|
+
const COOPERATIVE_BYTES = 16 * 1024 * 1024;
|
|
30
34
|
/**
|
|
31
35
|
* Both hash flavours answer in hex here; the branch exists so the seal loop can
|
|
32
36
|
* feed an injected resumable hasher and a fresh `createHash` identically. Note
|
|
@@ -35,10 +39,28 @@ const sha256 = (input) => createHash("sha256").update(input).digest("hex");
|
|
|
35
39
|
*/
|
|
36
40
|
const digestHex = (hash) => hash instanceof StreamingSha256 ? hash.digestHex() : hash.digest("hex");
|
|
37
41
|
/** `FileHandle.read` may come up short; a chunk boundary is not a suggestion. */
|
|
38
|
-
async function
|
|
42
|
+
async function waitForSignal(work, signal) {
|
|
43
|
+
if (!signal)
|
|
44
|
+
return work;
|
|
45
|
+
signal.throwIfAborted();
|
|
46
|
+
let cancel;
|
|
47
|
+
const aborted = new Promise((_resolve, reject) => {
|
|
48
|
+
cancel = () => reject(signal.reason instanceof Error ? signal.reason : new DOMException("The seal was cancelled", "AbortError"));
|
|
49
|
+
signal.addEventListener("abort", cancel, { once: true });
|
|
50
|
+
});
|
|
51
|
+
try {
|
|
52
|
+
return await Promise.race([work, aborted]);
|
|
53
|
+
}
|
|
54
|
+
finally {
|
|
55
|
+
if (cancel)
|
|
56
|
+
signal.removeEventListener("abort", cancel);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
async function readExactly(handle, into, length, position, signal) {
|
|
39
60
|
let filled = 0;
|
|
40
61
|
while (filled < length) {
|
|
41
|
-
|
|
62
|
+
signal?.throwIfAborted();
|
|
63
|
+
const { bytesRead } = await waitForSignal(handle.read(into, filled, length - filled, position + filled), signal);
|
|
42
64
|
if (bytesRead === 0)
|
|
43
65
|
throw new CryptoError("ciphertext_length_mismatch", `Expected ${length} bytes at offset ${position}; the file ended after ${filled}`);
|
|
44
66
|
filled += bytesRead;
|
|
@@ -53,6 +75,7 @@ async function readExactly(handle, into, length, position) {
|
|
|
53
75
|
* must happen after this resolves. On failure the destination is removed.
|
|
54
76
|
*/
|
|
55
77
|
export async function sealArchiveToFile(source, destPath, options) {
|
|
78
|
+
options.signal?.throwIfAborted();
|
|
56
79
|
const suite = options.suite ?? "chacha20-poly1305";
|
|
57
80
|
if (!SUITES.includes(suite))
|
|
58
81
|
throw new CryptoError("suite_unsupported", `Unsupported suite: ${suite}`);
|
|
@@ -64,8 +87,14 @@ export async function sealArchiveToFile(source, destPath, options) {
|
|
|
64
87
|
const noncePrefix = options.deterministic?.noncePrefix ?? randomBytes(4);
|
|
65
88
|
if (archiveKey.length !== KEY_BYTES)
|
|
66
89
|
throw new CryptoError("key_malformed", `An archive key must be ${KEY_BYTES} bytes`);
|
|
67
|
-
|
|
68
|
-
|
|
90
|
+
// Which compressor, and whether it sees the whole file or one chunk at a
|
|
91
|
+
// time. Whole-file is the one that matters: the duplicate screenshots this
|
|
92
|
+
// is here to collapse sit megabytes apart, so a per-chunk window never
|
|
93
|
+
// sees them. Measured on one 672 MB session: 4 MB chunks 2x, whole file 59x.
|
|
94
|
+
const codec = options.compression ?? "none";
|
|
95
|
+
const zstd = codec === "zstd" || codec === "zstd-chunk";
|
|
96
|
+
const compressed = codec === "gzip" || codec === "zstd";
|
|
97
|
+
const chunkCompressed = codec === "gzip-chunk" || codec === "zstd-chunk";
|
|
69
98
|
const start = source.start ?? 0;
|
|
70
99
|
const sourceBytes = (await stat(source.path)).size;
|
|
71
100
|
const end = Math.min(source.end ?? sourceBytes, sourceBytes);
|
|
@@ -101,16 +130,23 @@ export async function sealArchiveToFile(source, destPath, options) {
|
|
|
101
130
|
let originalBytes = 0;
|
|
102
131
|
try {
|
|
103
132
|
if (compressed) {
|
|
133
|
+
let nextYieldAt = COOPERATIVE_BYTES;
|
|
104
134
|
// Pass one: the caller's bytes are hashed on the way past, so the source
|
|
105
135
|
// is read exactly once even though the body is produced separately.
|
|
106
|
-
await pipeline(createReadStream(source.path, { start, end: Math.max(start, end - 1) }), async function* (chunks) {
|
|
136
|
+
await pipeline(createReadStream(source.path, { start, end: Math.max(start, end - 1), signal: options.signal }), async function* (chunks) {
|
|
107
137
|
for await (const piece of chunks) {
|
|
138
|
+
options.signal?.throwIfAborted();
|
|
108
139
|
originalHash.update(piece);
|
|
109
140
|
originalBytes += piece.length;
|
|
110
141
|
options.onProgress?.(originalBytes);
|
|
142
|
+
if (originalBytes >= nextYieldAt) {
|
|
143
|
+
nextYieldAt = originalBytes + COOPERATIVE_BYTES;
|
|
144
|
+
await options.onYield?.();
|
|
145
|
+
options.signal?.throwIfAborted();
|
|
146
|
+
}
|
|
111
147
|
yield piece;
|
|
112
148
|
}
|
|
113
|
-
}, createGzip({ level: 6 }), createWriteStream(bodyPath, { mode: 0o600 }));
|
|
149
|
+
}, zstd ? squeezeStream() : createGzip({ level: 6 }), createWriteStream(bodyPath, { mode: 0o600, signal: options.signal }), { signal: options.signal });
|
|
114
150
|
}
|
|
115
151
|
const bodyStart = compressed ? 0 : start;
|
|
116
152
|
const bodyBytes = compressed ? (await stat(bodyPath)).size : end - start;
|
|
@@ -120,12 +156,14 @@ export async function sealArchiveToFile(source, destPath, options) {
|
|
|
120
156
|
const ciphertextHash = createHash("sha256");
|
|
121
157
|
let storedBytes = 0;
|
|
122
158
|
const buffer = Buffer.allocUnsafe(Math.min(chunkBytes, Math.max(bodyBytes, 1)));
|
|
159
|
+
let nextSealYieldAt = COOPERATIVE_BYTES;
|
|
123
160
|
const body = await open(bodyPath, "r");
|
|
124
161
|
const dest = await open(destPath, "w", 0o600);
|
|
125
162
|
try {
|
|
126
163
|
for (let index = 0; index < totalChunks; index += 1) {
|
|
164
|
+
options.signal?.throwIfAborted();
|
|
127
165
|
const want = Math.max(0, Math.min(chunkBytes, bodyBytes - index * chunkBytes));
|
|
128
|
-
const slice = want === 0 ? buffer.subarray(0, 0) : await readExactly(body, buffer, want, bodyStart + index * chunkBytes);
|
|
166
|
+
const slice = want === 0 ? buffer.subarray(0, 0) : await readExactly(body, buffer, want, bodyStart + index * chunkBytes, options.signal);
|
|
129
167
|
const final = index === totalChunks - 1;
|
|
130
168
|
const nonce = chunkNonce(noncePrefix, index);
|
|
131
169
|
const cipher = aeadCipher(suite, archiveKey, nonce);
|
|
@@ -144,12 +182,17 @@ export async function sealArchiveToFile(source, destPath, options) {
|
|
|
144
182
|
// the restore, which is far too late if the source has been reclaimed
|
|
145
183
|
// in between. Refusing here keeps the archive and the hash the same
|
|
146
184
|
// object for the same reason the destination path checks its errno.
|
|
147
|
-
const { bytesWritten } = await dest.write(stored);
|
|
185
|
+
const { bytesWritten } = await waitForSignal(dest.write(stored), options.signal);
|
|
148
186
|
if (bytesWritten !== stored.length) {
|
|
149
187
|
throw new CryptoError("destination_write_failed", `Only ${bytesWritten} of ${stored.length} bytes of chunk ${index} reached ${destPath} — the volume is probably full`);
|
|
150
188
|
}
|
|
151
189
|
storedBytes += stored.length;
|
|
152
190
|
chunks.push({ index, plaintextBytes: slice.length, storedBytes: stored.length, sha256: sha256(stored), nonce: nonce.toString("base64"), final });
|
|
191
|
+
if ((index + 1) * chunkBytes >= nextSealYieldAt) {
|
|
192
|
+
nextSealYieldAt = (index + 1) * chunkBytes + COOPERATIVE_BYTES;
|
|
193
|
+
await options.onYield?.();
|
|
194
|
+
options.signal?.throwIfAborted();
|
|
195
|
+
}
|
|
153
196
|
}
|
|
154
197
|
}
|
|
155
198
|
finally {
|
|
@@ -160,7 +203,7 @@ export async function sealArchiveToFile(source, destPath, options) {
|
|
|
160
203
|
version: ENVELOPE_VERSION, archiveId, suite, chunkBytes, chunks,
|
|
161
204
|
manifest: {
|
|
162
205
|
plaintextSha256: bodyHash.digest("hex"), plaintextBytes: bodyBytes,
|
|
163
|
-
...(compressed ? { compression: "gzip", originalBytes, originalSha256: originalHash.digest("hex") } : {}),
|
|
206
|
+
...(compressed ? { compression: zstd ? "zstd" : "gzip", originalBytes, originalSha256: originalHash.digest("hex") } : {}),
|
|
164
207
|
createdAt: options.createdAt ?? new Date().toISOString(), adapter: options.adapter
|
|
165
208
|
},
|
|
166
209
|
wrappedKeys: wrapAll(archiveKey, options.recipients, suite, archiveId, options.deterministic)
|
|
@@ -216,12 +259,46 @@ function writeFailure(error, destPath) {
|
|
|
216
259
|
* a rejection can leave a partial file behind, which is why `restoreArchive`
|
|
217
260
|
* renames into place rather than writing the destination directly.
|
|
218
261
|
*/
|
|
219
|
-
|
|
262
|
+
/**
|
|
263
|
+
* `destPath` may be a Writable instead of a path. Verification needs to prove an
|
|
264
|
+
* archive decrypts to its recorded bytes, and materialising the plaintext to do
|
|
265
|
+
* so costs as much free disk as the original — on the nearly-full disk that
|
|
266
|
+
* makes someone reach for this product in the first place, that is exactly the
|
|
267
|
+
* space they do not have, so the proof became impossible precisely when it
|
|
268
|
+
* mattered. Every integrity check below runs the same either way; only the
|
|
269
|
+
* destination changes.
|
|
270
|
+
*/
|
|
271
|
+
export async function openArchiveToFile(envelope, ciphertextPath, destPath, unlock, options = {}) {
|
|
272
|
+
const sink = () => typeof destPath === "string" ? createWriteStream(destPath, { mode: 0o600 }) : destPath;
|
|
273
|
+
const destLabel = typeof destPath === "string" ? destPath : "(stream)";
|
|
220
274
|
const storedOnDisk = (await stat(ciphertextPath)).size;
|
|
221
275
|
validateEnvelope(envelope, storedOnDisk);
|
|
276
|
+
if (options.maxStoredChunkBytes !== undefined
|
|
277
|
+
&& (!Number.isSafeInteger(options.maxStoredChunkBytes) || options.maxStoredChunkBytes <= 0)) {
|
|
278
|
+
throw new CryptoError("resource_limit_invalid", "maxStoredChunkBytes must be a positive safe integer");
|
|
279
|
+
}
|
|
280
|
+
if (options.maxOriginalBytes !== undefined
|
|
281
|
+
&& (!Number.isSafeInteger(options.maxOriginalBytes) || options.maxOriginalBytes < 0)) {
|
|
282
|
+
throw new CryptoError("resource_limit_invalid", "maxOriginalBytes must be a non-negative safe integer");
|
|
283
|
+
}
|
|
284
|
+
if (options.maxExpandedChunkBytes !== undefined
|
|
285
|
+
&& (!Number.isSafeInteger(options.maxExpandedChunkBytes) || options.maxExpandedChunkBytes < 0)) {
|
|
286
|
+
throw new CryptoError("resource_limit_invalid", "maxExpandedChunkBytes must be a non-negative safe integer");
|
|
287
|
+
}
|
|
288
|
+
if (options.maxStoredChunkBytes !== undefined
|
|
289
|
+
&& envelope.chunks.some((chunk) => chunk.storedBytes > options.maxStoredChunkBytes)) {
|
|
290
|
+
throw new CryptoError("resource_limit_exceeded", "A stored archive chunk exceeds the configured opening limit");
|
|
291
|
+
}
|
|
222
292
|
const compression = envelope.manifest.compression;
|
|
223
|
-
if (compression !== undefined && compression !== "gzip" && compression !== "gzip-chunk")
|
|
293
|
+
if (compression !== undefined && compression !== "gzip" && compression !== "gzip-chunk" && compression !== "zstd" && compression !== "zstd-chunk")
|
|
224
294
|
throw new CryptoError("envelope_malformed", "Unsupported compression");
|
|
295
|
+
// The archive says which compressor sealed it. Everything written before
|
|
296
|
+
// zstd existed says gzip and keeps opening through the gzip path.
|
|
297
|
+
const wasZstd = compression === "zstd" || compression === "zstd-chunk";
|
|
298
|
+
const declaredOriginalBytes = compression === undefined ? envelope.manifest.plaintextBytes : envelope.manifest.originalBytes;
|
|
299
|
+
if (options.maxOriginalBytes !== undefined && declaredOriginalBytes > options.maxOriginalBytes) {
|
|
300
|
+
throw new CryptoError("resource_limit_exceeded", "The archive's declared original size exceeds the configured opening limit");
|
|
301
|
+
}
|
|
225
302
|
const archiveKey = unwrapArchiveKey(envelope.wrappedKeys, envelope.suite, envelope.archiveId, unlock);
|
|
226
303
|
if (archiveKey.length !== KEY_BYTES) {
|
|
227
304
|
zeroize(archiveKey);
|
|
@@ -262,40 +339,80 @@ export async function openArchiveToFile(envelope, ciphertextPath, destPath, unlo
|
|
|
262
339
|
if (bodyDigest !== envelope.manifest.plaintextSha256)
|
|
263
340
|
throw new CryptoError("plaintext_hash_mismatch", "Decrypted bytes do not match the manifest hash");
|
|
264
341
|
}
|
|
342
|
+
function accountOriginal(piece) {
|
|
343
|
+
const next = originalBytes + piece.length;
|
|
344
|
+
if (!Number.isSafeInteger(next) || (options.maxOriginalBytes !== undefined && next > options.maxOriginalBytes)) {
|
|
345
|
+
throw new CryptoError("resource_limit_exceeded", "Decompressed archive bytes exceed the configured opening limit");
|
|
346
|
+
}
|
|
347
|
+
originalHash.update(piece);
|
|
348
|
+
originalBytes = next;
|
|
349
|
+
}
|
|
265
350
|
async function* measure(chunks) {
|
|
266
351
|
for await (const piece of chunks) {
|
|
267
|
-
|
|
268
|
-
originalBytes += piece.length;
|
|
352
|
+
accountOriginal(piece);
|
|
269
353
|
yield piece;
|
|
270
354
|
}
|
|
271
355
|
}
|
|
272
356
|
try {
|
|
273
|
-
if (compression === "gzip-chunk") {
|
|
357
|
+
if (compression === "gzip-chunk" || compression === "zstd-chunk") {
|
|
274
358
|
// Each authenticated piece is exactly one chunk's compressed slice —
|
|
275
359
|
// undone on its own, in chunk order. This is the same access pattern a
|
|
276
360
|
// ranged fetch will use, just walked start to end.
|
|
277
361
|
async function* perChunk(chunks) {
|
|
362
|
+
let index = 0;
|
|
278
363
|
for await (const piece of chunks) {
|
|
279
|
-
let
|
|
280
|
-
|
|
281
|
-
|
|
364
|
+
let chunkBytes = 0;
|
|
365
|
+
const accountChunk = (slice) => {
|
|
366
|
+
const next = chunkBytes + slice.length;
|
|
367
|
+
if (!Number.isSafeInteger(next)
|
|
368
|
+
|| (options.maxExpandedChunkBytes !== undefined && next > options.maxExpandedChunkBytes)) {
|
|
369
|
+
throw new CryptoError("resource_limit_exceeded", "One decompressed archive chunk exceeds the configured opening limit");
|
|
370
|
+
}
|
|
371
|
+
chunkBytes = next;
|
|
372
|
+
accountOriginal(slice);
|
|
373
|
+
};
|
|
374
|
+
if (options.streamChunkDecompression || options.maxOriginalBytes !== undefined
|
|
375
|
+
|| options.maxExpandedChunkBytes !== undefined) {
|
|
376
|
+
try {
|
|
377
|
+
const expanded = Readable.from([piece]).pipe(wasZstd ? unsqueezeStream() : createGunzip());
|
|
378
|
+
for await (const chunk of expanded) {
|
|
379
|
+
const slice = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
380
|
+
accountChunk(slice);
|
|
381
|
+
yield slice;
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
catch (error) {
|
|
385
|
+
if (error instanceof CryptoError)
|
|
386
|
+
throw error;
|
|
387
|
+
throw new CryptoError("plaintext_hash_mismatch", "Decompression failed after authentication");
|
|
388
|
+
}
|
|
282
389
|
}
|
|
283
|
-
|
|
284
|
-
|
|
390
|
+
else {
|
|
391
|
+
let slice;
|
|
392
|
+
try {
|
|
393
|
+
slice = wasZstd ? unsqueezeSync(piece) : gunzipSync(piece);
|
|
394
|
+
}
|
|
395
|
+
catch {
|
|
396
|
+
throw new CryptoError("plaintext_hash_mismatch", "Decompression failed after authentication");
|
|
397
|
+
}
|
|
398
|
+
accountChunk(slice);
|
|
399
|
+
yield slice;
|
|
285
400
|
}
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
401
|
+
const declaredRawBytes = envelope.chunks[index]?.rawBytes;
|
|
402
|
+
if (declaredRawBytes !== undefined && chunkBytes !== declaredRawBytes) {
|
|
403
|
+
throw new CryptoError("plaintext_length_mismatch", `Decompressed chunk ${index} does not match its recorded original length`);
|
|
404
|
+
}
|
|
405
|
+
index += 1;
|
|
289
406
|
}
|
|
290
407
|
}
|
|
291
408
|
try {
|
|
292
|
-
await pipeline(authenticatedBody(), perChunk,
|
|
409
|
+
await pipeline(authenticatedBody(), perChunk, sink());
|
|
293
410
|
}
|
|
294
411
|
catch (error) {
|
|
295
412
|
if (error instanceof CryptoError)
|
|
296
413
|
throw error;
|
|
297
414
|
if (isSystemError(error))
|
|
298
|
-
throw writeFailure(error,
|
|
415
|
+
throw writeFailure(error, destLabel);
|
|
299
416
|
throw error;
|
|
300
417
|
}
|
|
301
418
|
if (originalBytes !== envelope.manifest.originalBytes)
|
|
@@ -305,9 +422,9 @@ export async function openArchiveToFile(envelope, ciphertextPath, destPath, unlo
|
|
|
305
422
|
throw new CryptoError("plaintext_hash_mismatch", "Decompressed bytes do not match the manifest hash");
|
|
306
423
|
return { bytes: originalBytes, sha256: digest };
|
|
307
424
|
}
|
|
308
|
-
if (compression === "gzip") {
|
|
425
|
+
if (compression === "gzip" || compression === "zstd") {
|
|
309
426
|
try {
|
|
310
|
-
await pipeline(authenticatedBody(), createGunzip(), measure,
|
|
427
|
+
await pipeline(authenticatedBody(), wasZstd ? unsqueezeStream() : createGunzip(), measure, sink());
|
|
311
428
|
}
|
|
312
429
|
catch (error) {
|
|
313
430
|
// A zlib failure here is not a tampering signal — every byte fed to the
|
|
@@ -316,7 +433,7 @@ export async function openArchiveToFile(envelope, ciphertextPath, destPath, unlo
|
|
|
316
433
|
if (error instanceof CryptoError)
|
|
317
434
|
throw error;
|
|
318
435
|
if (isSystemError(error))
|
|
319
|
-
throw writeFailure(error,
|
|
436
|
+
throw writeFailure(error, destLabel);
|
|
320
437
|
throw new CryptoError("plaintext_hash_mismatch", "Decompression failed after authentication");
|
|
321
438
|
}
|
|
322
439
|
// The archive is honoured only once the caller's own bytes are proven, not
|
|
@@ -329,13 +446,13 @@ export async function openArchiveToFile(envelope, ciphertextPath, destPath, unlo
|
|
|
329
446
|
return { bytes: originalBytes, sha256: digest };
|
|
330
447
|
}
|
|
331
448
|
try {
|
|
332
|
-
await pipeline(authenticatedBody(),
|
|
449
|
+
await pipeline(authenticatedBody(), sink());
|
|
333
450
|
}
|
|
334
451
|
catch (error) {
|
|
335
452
|
if (error instanceof CryptoError)
|
|
336
453
|
throw error;
|
|
337
454
|
if (isSystemError(error))
|
|
338
|
-
throw writeFailure(error,
|
|
455
|
+
throw writeFailure(error, destLabel);
|
|
339
456
|
throw error;
|
|
340
457
|
}
|
|
341
458
|
return { bytes: bodyBytes, sha256: bodyDigest };
|
|
@@ -356,7 +473,8 @@ export async function openArchiveToFile(envelope, ciphertextPath, destPath, unlo
|
|
|
356
473
|
*
|
|
357
474
|
* An offset past the end of the file is simply absent from the result.
|
|
358
475
|
*/
|
|
359
|
-
export async function hashFilePrefixes(path, offsets) {
|
|
476
|
+
export async function hashFilePrefixes(path, offsets, onProgress, signal, onYield) {
|
|
477
|
+
signal?.throwIfAborted();
|
|
360
478
|
const wanted = [...new Set(offsets)].filter((offset) => offset > 0).sort((a, b) => a - b);
|
|
361
479
|
const digests = new Map();
|
|
362
480
|
if (wanted.length === 0)
|
|
@@ -367,7 +485,9 @@ export async function hashFilePrefixes(path, offsets) {
|
|
|
367
485
|
const limit = wanted[wanted.length - 1];
|
|
368
486
|
let read = 0;
|
|
369
487
|
let next = 0;
|
|
370
|
-
|
|
488
|
+
let nextYield = COOPERATIVE_BYTES;
|
|
489
|
+
for await (const piece of createReadStream(path, { start: 0, end: limit - 1, signal })) {
|
|
490
|
+
signal?.throwIfAborted();
|
|
371
491
|
const buffer = piece;
|
|
372
492
|
let offset = 0;
|
|
373
493
|
while (offset < buffer.length && next < wanted.length) {
|
|
@@ -378,6 +498,17 @@ export async function hashFilePrefixes(path, offsets) {
|
|
|
378
498
|
while (next < wanted.length && wanted[next] === read)
|
|
379
499
|
digests.set(wanted[next++], hash.copy().digest("hex"));
|
|
380
500
|
}
|
|
501
|
+
onProgress?.(read);
|
|
502
|
+
if (read >= nextYield) {
|
|
503
|
+
nextYield = read + COOPERATIVE_BYTES;
|
|
504
|
+
// A cached multi-gigabyte file can keep the stream iterator's promises
|
|
505
|
+
// continuously ready. Explicitly visit the timers/check phases so the
|
|
506
|
+
// daemon heartbeat and queue lease are not starved by useful hashing.
|
|
507
|
+
if (onYield)
|
|
508
|
+
await onYield();
|
|
509
|
+
else
|
|
510
|
+
await yieldToEventLoop();
|
|
511
|
+
}
|
|
381
512
|
}
|
|
382
513
|
return digests;
|
|
383
514
|
}
|
|
@@ -388,15 +519,26 @@ export async function hashFilePrefixes(path, offsets) {
|
|
|
388
519
|
* live file and comparing it with what an archive recorded is the only evidence
|
|
389
520
|
* that the archive's bytes are still a prefix of it.
|
|
390
521
|
*/
|
|
391
|
-
export async function hashFileRange(path, start = 0, end) {
|
|
522
|
+
export async function hashFileRange(path, start = 0, end, onProgress, signal, onYield) {
|
|
523
|
+
signal?.throwIfAborted();
|
|
392
524
|
const size = (await stat(path)).size;
|
|
393
525
|
const last = Math.min(end ?? size, size);
|
|
394
526
|
const hash = createHash("sha256");
|
|
395
527
|
let bytes = 0;
|
|
528
|
+
let nextYield = COOPERATIVE_BYTES;
|
|
396
529
|
if (last > start) {
|
|
397
|
-
for await (const piece of createReadStream(path, { start, end: last - 1 })) {
|
|
530
|
+
for await (const piece of createReadStream(path, { start, end: last - 1, signal })) {
|
|
531
|
+
signal?.throwIfAborted();
|
|
398
532
|
hash.update(piece);
|
|
399
533
|
bytes += piece.length;
|
|
534
|
+
onProgress?.(bytes);
|
|
535
|
+
if (bytes >= nextYield) {
|
|
536
|
+
nextYield = bytes + COOPERATIVE_BYTES;
|
|
537
|
+
if (onYield)
|
|
538
|
+
await onYield();
|
|
539
|
+
else
|
|
540
|
+
await yieldToEventLoop();
|
|
541
|
+
}
|
|
400
542
|
}
|
|
401
543
|
}
|
|
402
544
|
return { sha256: hash.digest("hex"), bytes };
|
|
@@ -409,6 +551,7 @@ export async function hashFileRange(path, start = 0, end) {
|
|
|
409
551
|
* in memory, full stop.
|
|
410
552
|
*/
|
|
411
553
|
export async function sealChunksToSink(source, sink, options) {
|
|
554
|
+
options.signal?.throwIfAborted();
|
|
412
555
|
const suite = options.suite ?? "chacha20-poly1305";
|
|
413
556
|
const chunkBytes = options.chunkBytes ?? DEFAULT_CHUNK_BYTES;
|
|
414
557
|
const archiveId = options.archiveId ?? randomUUID();
|
|
@@ -416,6 +559,9 @@ export async function sealChunksToSink(source, sink, options) {
|
|
|
416
559
|
const archiveKey = options.archiveKey ?? randomBytes(KEY_BYTES);
|
|
417
560
|
const noncePrefix = options.noncePrefix ?? randomBytes(4);
|
|
418
561
|
const firstChunkIndex = options.firstChunkIndex ?? 0;
|
|
562
|
+
// Per-chunk compression, so the window is one chunk wide either way; zstd
|
|
563
|
+
// still beats gzip here, just not by the margin the whole-file path gets.
|
|
564
|
+
const zstd = (options.compression ?? "none") === "zstd-chunk";
|
|
419
565
|
if (!Number.isInteger(firstChunkIndex) || firstChunkIndex < 0)
|
|
420
566
|
throw new CryptoError("invalid_range", "firstChunkIndex must be a non-negative integer");
|
|
421
567
|
const start = source.start ?? 0;
|
|
@@ -433,18 +579,31 @@ export async function sealChunksToSink(source, sink, options) {
|
|
|
433
579
|
let originalBytes = 0;
|
|
434
580
|
let bodyBytes = 0;
|
|
435
581
|
let storedBytesTotal = 0;
|
|
582
|
+
let nextYield = COOPERATIVE_BYTES;
|
|
436
583
|
const buffer = Buffer.allocUnsafe(Math.min(chunkBytes, Math.max(raw, 1)));
|
|
437
584
|
const src = await open(source.path, "r");
|
|
438
585
|
try {
|
|
439
586
|
for (let position = 0; position < localChunks; position += 1) {
|
|
587
|
+
options.signal?.throwIfAborted();
|
|
440
588
|
const index = firstChunkIndex + position;
|
|
441
589
|
const want = Math.max(0, Math.min(chunkBytes, raw - position * chunkBytes));
|
|
442
|
-
const slice = want === 0 ? buffer.subarray(0, 0) : await readExactly(src, buffer, want, start + position * chunkBytes);
|
|
590
|
+
const slice = want === 0 ? buffer.subarray(0, 0) : await readExactly(src, buffer, want, start + position * chunkBytes, options.signal);
|
|
443
591
|
originalHash.update(slice);
|
|
444
592
|
originalBytes += slice.length;
|
|
445
593
|
options.onProgress?.(originalBytes);
|
|
446
594
|
options.onPlaintext?.(index, slice);
|
|
447
|
-
const packed = gzipSync(slice, { level: 6 });
|
|
595
|
+
const packed = zstd ? squeezeSync(slice) : gzipSync(slice, { level: 6 });
|
|
596
|
+
// Automatic callers use reviewed 4 MiB chunks and account every chunk;
|
|
597
|
+
// manual callers still get the ordinary 16 MiB event-loop checkpoint.
|
|
598
|
+
// This runs after compression so the duty-cycle controller measures the
|
|
599
|
+
// synchronous work it is repaying before the next chunk starts.
|
|
600
|
+
if (options.onYield)
|
|
601
|
+
await options.onYield();
|
|
602
|
+
else if (originalBytes >= nextYield) {
|
|
603
|
+
nextYield = originalBytes + COOPERATIVE_BYTES;
|
|
604
|
+
await yieldToEventLoop();
|
|
605
|
+
}
|
|
606
|
+
options.signal?.throwIfAborted();
|
|
448
607
|
const final = index === totalChunks - 1;
|
|
449
608
|
const nonce = chunkNonce(noncePrefix, index);
|
|
450
609
|
const cipher = aeadCipher(suite, archiveKey, nonce);
|
|
@@ -453,7 +612,7 @@ export async function sealChunksToSink(source, sink, options) {
|
|
|
453
612
|
bodyHash.update(packed);
|
|
454
613
|
bodyBytes += packed.length;
|
|
455
614
|
ciphertextHash.update(stored);
|
|
456
|
-
await sink.write(stored);
|
|
615
|
+
await waitForSignal(sink.write(stored), options.signal);
|
|
457
616
|
storedBytesTotal += stored.length;
|
|
458
617
|
chunks.push({ index, plaintextBytes: packed.length, rawBytes: slice.length, storedBytes: stored.length, sha256: createHash("sha256").update(stored).digest("hex"), nonce: nonce.toString("base64"), final });
|
|
459
618
|
}
|
|
@@ -466,7 +625,7 @@ export async function sealChunksToSink(source, sink, options) {
|
|
|
466
625
|
version: ENVELOPE_VERSION, archiveId, suite, chunkBytes, chunks,
|
|
467
626
|
manifest: {
|
|
468
627
|
plaintextSha256: bodyHash.digest("hex"), plaintextBytes: bodyBytes,
|
|
469
|
-
compression: "gzip-chunk", originalBytes, originalSha256,
|
|
628
|
+
compression: zstd ? "zstd-chunk" : "gzip-chunk", originalBytes, originalSha256,
|
|
470
629
|
createdAt: options.createdAt ?? new Date().toISOString(), adapter: options.adapter
|
|
471
630
|
},
|
|
472
631
|
wrappedKeys: wrapAll(archiveKey, options.recipients, suite, archiveId, options.deterministic)
|