sealkeep 0.8.1 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. package/ARCHITECTURE.md +163 -14
  2. package/CHANGELOG.md +244 -1
  3. package/CONTROL_PLANE.md +2 -2
  4. package/LICENSE +1 -1
  5. package/README.md +108 -22
  6. package/THIRD_PARTY.md +2 -2
  7. package/THREAT_MODEL.md +23 -4
  8. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/chunk-access.d.ts +26 -4
  9. package/dist/packages/sealkeep-crypto/src/chunk-access.js +219 -0
  10. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/envelope.d.ts +11 -1
  11. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/envelope.js +58 -21
  12. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/format.d.ts +2 -2
  13. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/index.d.ts +1 -0
  14. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/index.js +1 -0
  15. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/recipients.js +4 -4
  16. package/dist/packages/sealkeep-crypto/src/squeeze.d.ts +6 -0
  17. package/dist/packages/sealkeep-crypto/src/squeeze.js +39 -0
  18. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/stream.d.ts +36 -3
  19. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/stream.js +196 -37
  20. package/dist/site/index.html +1808 -1904
  21. package/dist/site/llms.txt +67 -0
  22. package/dist/site/trust/architecture-data-flow.html +53 -0
  23. package/dist/site/trust/audit-roadmap.html +37 -0
  24. package/dist/site/trust/deployment-responsibility.html +11 -0
  25. package/dist/site/trust/dpa-sample.html +30 -0
  26. package/dist/site/trust/release-provenance.html +21 -0
  27. package/dist/site/trust/subprocessors.html +15 -0
  28. package/dist/site/trust/threat-model.html +34 -0
  29. package/dist/site/trust/toms.html +41 -0
  30. package/dist/site/trust-document.css +32 -0
  31. package/dist/site/trust.html +73 -0
  32. package/dist/site/visual/assets/index-8Nxnkc7o.js +207 -0
  33. package/dist/site/visual/assets/index-oxLy1bpy.css +1 -0
  34. package/dist/site/visual/index.html +18 -0
  35. package/dist/site.zip +0 -0
  36. package/dist/src/activity.d.ts +9 -0
  37. package/dist/src/activity.js +90 -1
  38. package/dist/src/adapters.d.ts +175 -5
  39. package/dist/src/adapters.js +961 -73
  40. package/dist/src/agent-context.d.ts +135 -0
  41. package/dist/src/agent-context.js +1059 -0
  42. package/dist/src/archive-copies.d.ts +47 -0
  43. package/dist/src/archive-copies.js +179 -0
  44. package/dist/src/audit.d.ts +1 -1
  45. package/dist/src/audit.js +29 -4
  46. package/dist/src/autopilot.d.ts +52 -7
  47. package/dist/src/autopilot.js +143 -25
  48. package/dist/src/background-bandwidth.d.ts +46 -0
  49. package/dist/src/background-bandwidth.js +301 -0
  50. package/dist/src/background-cpu.d.ts +82 -0
  51. package/dist/src/background-cpu.js +212 -0
  52. package/dist/src/background-worker-error.d.ts +12 -0
  53. package/dist/src/background-worker-error.js +18 -0
  54. package/dist/src/branding.d.ts +24 -1
  55. package/dist/src/branding.js +31 -1
  56. package/dist/src/bridge.d.ts +233 -0
  57. package/dist/src/bridge.js +604 -0
  58. package/dist/src/byte-stream.d.ts +91 -0
  59. package/dist/src/byte-stream.js +385 -0
  60. package/dist/src/chunk-store.d.ts +41 -8
  61. package/dist/src/chunk-store.js +161 -65
  62. package/dist/src/cli.js +1746 -166
  63. package/dist/src/cloud.d.ts +841 -31
  64. package/dist/src/cloud.js +3196 -277
  65. package/dist/src/context-background.d.ts +37 -0
  66. package/dist/src/context-background.js +309 -0
  67. package/dist/src/context-drain-child.d.ts +1 -0
  68. package/dist/src/context-drain-child.js +98 -0
  69. package/dist/src/context-reader.d.ts +118 -0
  70. package/dist/src/context-reader.js +447 -0
  71. package/dist/src/control-plane/auth.d.ts +32 -4
  72. package/dist/src/control-plane/auth.js +85 -24
  73. package/dist/src/control-plane/server.js +19 -6
  74. package/dist/src/control-plane.d.ts +17 -1
  75. package/dist/src/control-plane.js +32 -6
  76. package/dist/src/crypto.d.ts +1 -1
  77. package/dist/src/crypto.js +5 -5
  78. package/dist/src/daemon-lease.d.ts +70 -0
  79. package/dist/src/daemon-lease.js +420 -0
  80. package/dist/src/daemon.d.ts +94 -1
  81. package/dist/src/daemon.js +1082 -105
  82. package/dist/src/darwin-service-policy.d.ts +41 -0
  83. package/dist/src/darwin-service-policy.js +60 -0
  84. package/dist/src/dashboard-cli.js +15 -15
  85. package/dist/src/device-authorization.d.ts +37 -0
  86. package/dist/src/device-authorization.js +199 -0
  87. package/dist/src/device-enrollment.d.ts +91 -0
  88. package/dist/src/device-enrollment.js +349 -0
  89. package/dist/src/disk.d.ts +17 -12
  90. package/dist/src/disk.js +43 -17
  91. package/dist/src/doctor.d.ts +35 -1
  92. package/dist/src/doctor.js +316 -41
  93. package/dist/src/durable-ticket-lock.d.ts +24 -0
  94. package/dist/src/durable-ticket-lock.js +232 -0
  95. package/dist/src/enroll.d.ts +1 -1
  96. package/dist/src/enroll.js +13 -7
  97. package/dist/src/env.d.ts +10 -1
  98. package/dist/src/env.js +11 -3
  99. package/dist/src/errors.d.ts +8 -8
  100. package/dist/src/errors.js +6 -6
  101. package/dist/src/flush.d.ts +12 -0
  102. package/dist/src/flush.js +37 -0
  103. package/dist/src/heartbeat.d.ts +86 -12
  104. package/dist/src/heartbeat.js +415 -29
  105. package/dist/src/index-background-watchdog.d.ts +1 -0
  106. package/dist/src/index-background-watchdog.js +94 -0
  107. package/dist/src/index-background-work.d.ts +21 -0
  108. package/dist/src/index-background-work.js +25 -0
  109. package/dist/src/index-background.d.ts +64 -0
  110. package/dist/src/index-background.js +394 -0
  111. package/dist/src/index-build-child.d.ts +1 -0
  112. package/dist/src/index-build-child.js +109 -0
  113. package/dist/src/index-manifest.d.ts +52 -0
  114. package/dist/src/index-manifest.js +444 -0
  115. package/dist/src/index-publication-proof.d.ts +84 -0
  116. package/dist/src/index-publication-proof.js +380 -0
  117. package/dist/src/index-publication-state.d.ts +149 -0
  118. package/dist/src/index-publication-state.js +696 -0
  119. package/dist/src/index-publication-verifier.d.ts +89 -0
  120. package/dist/src/index-publication-verifier.js +341 -0
  121. package/dist/src/index-publish.d.ts +62 -0
  122. package/dist/src/index-publish.js +540 -0
  123. package/dist/src/index-scratch-cleanup.d.ts +19 -0
  124. package/dist/src/index-scratch-cleanup.js +166 -0
  125. package/dist/src/index-segment-types.d.ts +132 -0
  126. package/dist/src/index-segment-types.js +21 -0
  127. package/dist/src/index-segments.d.ts +9 -0
  128. package/dist/src/index-segments.js +516 -0
  129. package/dist/src/index-store.d.ts +123 -0
  130. package/dist/src/index-store.js +495 -0
  131. package/dist/src/index-sync.d.ts +91 -15
  132. package/dist/src/index-sync.js +286 -53
  133. package/dist/src/index-upgrade-publication.d.ts +30 -0
  134. package/dist/src/index-upgrade-publication.js +179 -0
  135. package/dist/src/integration-manager.d.ts +32 -0
  136. package/dist/src/integration-manager.js +394 -0
  137. package/dist/src/leakscan.js +1 -1
  138. package/dist/src/local-api.d.ts +56 -15
  139. package/dist/src/local-api.js +4979 -541
  140. package/dist/src/machine-settings.d.ts +51 -0
  141. package/dist/src/machine-settings.js +166 -0
  142. package/dist/src/managed-chunks.d.ts +5 -2
  143. package/dist/src/managed-chunks.js +14 -14
  144. package/dist/src/mcp-install.d.ts +11 -9
  145. package/dist/src/mcp-install.js +73 -28
  146. package/dist/src/mcp-workspace.d.ts +18 -0
  147. package/dist/src/mcp-workspace.js +50 -0
  148. package/dist/src/mcp.js +294 -25
  149. package/dist/src/migrate.d.ts +1 -0
  150. package/dist/src/migrate.js +33 -5
  151. package/dist/src/notify.d.ts +1 -1
  152. package/dist/src/notify.js +5 -5
  153. package/dist/src/offload.d.ts +201 -14
  154. package/dist/src/offload.js +1848 -140
  155. package/dist/src/onboarding.d.ts +8 -1
  156. package/dist/src/onboarding.js +4 -4
  157. package/dist/src/packages.d.ts +2 -2
  158. package/dist/src/packages.js +10 -2
  159. package/dist/src/passkey.d.ts +0 -1
  160. package/dist/src/passkey.js +2 -7
  161. package/dist/src/password-lock.d.ts +2 -2
  162. package/dist/src/password-lock.js +6 -6
  163. package/dist/src/paths.d.ts +2 -0
  164. package/dist/src/paths.js +2 -0
  165. package/dist/src/presence.d.ts +86 -0
  166. package/dist/src/presence.js +240 -0
  167. package/dist/src/progress-deadline.d.ts +21 -0
  168. package/dist/src/progress-deadline.js +91 -0
  169. package/dist/src/project-repair.d.ts +55 -0
  170. package/dist/src/project-repair.js +131 -0
  171. package/dist/src/providers/gcs.d.ts +28 -7
  172. package/dist/src/providers/gcs.js +35 -24
  173. package/dist/src/providers/gdrive.d.ts +71 -8
  174. package/dist/src/providers/gdrive.js +223 -50
  175. package/dist/src/providers/index.d.ts +11 -3
  176. package/dist/src/providers/index.js +30 -10
  177. package/dist/src/providers/s3.d.ts +30 -8
  178. package/dist/src/providers/s3.js +41 -30
  179. package/dist/src/providers/safe-storage-fetch.d.ts +12 -0
  180. package/dist/src/providers/safe-storage-fetch.js +72 -0
  181. package/dist/src/queue.d.ts +136 -19
  182. package/dist/src/queue.js +862 -96
  183. package/dist/src/reclaim-transaction.d.ts +156 -0
  184. package/dist/src/reclaim-transaction.js +1027 -0
  185. package/dist/src/recovery-codes.d.ts +32 -0
  186. package/dist/src/recovery-codes.js +338 -0
  187. package/dist/src/recovery.js +12 -9
  188. package/dist/src/rehydrate.d.ts +25 -22
  189. package/dist/src/rehydrate.js +319 -23
  190. package/dist/src/restore.d.ts +57 -4
  191. package/dist/src/restore.js +272 -36
  192. package/dist/src/resume-stub.d.ts +92 -0
  193. package/dist/src/resume-stub.js +417 -0
  194. package/dist/src/retention.d.ts +98 -7
  195. package/dist/src/retention.js +1066 -62
  196. package/dist/src/rotate.js +3 -3
  197. package/dist/src/search.d.ts +566 -8
  198. package/dist/src/search.js +5897 -290
  199. package/dist/src/secrets.d.ts +51 -7
  200. package/dist/src/secrets.js +316 -24
  201. package/dist/src/service.d.ts +49 -11
  202. package/dist/src/service.js +776 -35
  203. package/dist/src/share.js +3 -3
  204. package/dist/src/shared-spaces.d.ts +98 -0
  205. package/dist/src/shared-spaces.js +214 -0
  206. package/dist/src/source-reader.d.ts +73 -0
  207. package/dist/src/source-reader.js +715 -0
  208. package/dist/src/spool.d.ts +1 -1
  209. package/dist/src/spool.js +1 -1
  210. package/dist/src/start-tui.js +2 -1
  211. package/dist/src/start.js +2 -2
  212. package/dist/src/storage-endpoint.d.ts +21 -0
  213. package/dist/src/storage-endpoint.js +122 -0
  214. package/dist/src/storage-setup.js +12 -12
  215. package/dist/src/storage-targets.d.ts +109 -6
  216. package/dist/src/storage-targets.js +975 -67
  217. package/dist/src/stream-to-cloud.d.ts +5 -1
  218. package/dist/src/stream-to-cloud.js +34 -14
  219. package/dist/src/sync-rules.d.ts +31 -6
  220. package/dist/src/sync-rules.js +153 -14
  221. package/dist/src/team-backfill-scheduling.d.ts +8 -0
  222. package/dist/src/team-backfill-scheduling.js +33 -0
  223. package/dist/src/team-backfill.d.ts +116 -0
  224. package/dist/src/team-backfill.js +1429 -0
  225. package/dist/src/team-index-cache.d.ts +16 -0
  226. package/dist/src/team-index-cache.js +152 -0
  227. package/dist/src/team-offboarding.d.ts +38 -0
  228. package/dist/src/team-offboarding.js +1043 -0
  229. package/dist/src/team-presence.d.ts +127 -0
  230. package/dist/src/team-presence.js +904 -0
  231. package/dist/src/team-publication-policy.d.ts +20 -0
  232. package/dist/src/team-publication-policy.js +140 -0
  233. package/dist/src/team-realtime.d.ts +68 -0
  234. package/dist/src/team-realtime.js +816 -0
  235. package/dist/src/team-source-facts-cache.d.ts +23 -0
  236. package/dist/src/team-source-facts-cache.js +255 -0
  237. package/dist/src/trash.d.ts +1 -1
  238. package/dist/src/trash.js +2 -2
  239. package/dist/src/tui.js +11 -12
  240. package/dist/src/types.d.ts +173 -7
  241. package/dist/src/types.js +20 -0
  242. package/dist/src/ui-server.d.ts +163 -35
  243. package/dist/src/ui-server.js +712 -72
  244. package/dist/src/ui.d.ts +1 -2
  245. package/dist/src/ui.js +1 -2
  246. package/dist/src/upload.d.ts +27 -0
  247. package/dist/src/upload.js +383 -43
  248. package/dist/src/vault.d.ts +235 -29
  249. package/dist/src/vault.js +1783 -190
  250. package/dist/src/watcher.d.ts +7 -1
  251. package/dist/src/watcher.js +198 -55
  252. package/dist/src/worker.d.ts +27 -3
  253. package/dist/src/worker.js +274 -55
  254. package/package.json +33 -12
  255. package/scripts/native-reboot-rehearsal.mjs +90 -0
  256. package/web/app.js +6032 -343
  257. package/web/bootstrap.js +17 -0
  258. package/web/index.html +255 -57
  259. package/web/rail.js +317 -40
  260. package/web/retention.html +2 -2
  261. package/web/rules-view.js +188 -16
  262. package/web/sessions-view.js +485 -62
  263. package/web/sessions.html +2 -2
  264. package/web/setup-api.js +152 -29
  265. package/web/setup-logic.js +68 -9
  266. package/web/setup.html +113 -44
  267. package/web/setup.js +604 -71
  268. package/web/style.css +513 -98
  269. package/dist/packages/vaultline-crypto/src/chunk-access.js +0 -93
  270. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/aead.d.ts +0 -0
  271. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/aead.js +0 -0
  272. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/format.js +0 -0
  273. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/recipients.d.ts +0 -0
  274. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/sha256-stream.d.ts +0 -0
  275. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/sha256-stream.js +0 -0
@@ -0,0 +1,91 @@
1
+ import { createHash } from "node:crypto";
2
+ import { type FileHandle } from "node:fs/promises";
3
+ import { Readable } from "node:stream";
4
+ /**
5
+ * One deliberately small I/O window for archive copies. Providers may keep a
6
+ * part of their own (normally 8 MiB), but reading a multi-gigabyte file never
7
+ * creates a buffer proportional to the file.
8
+ */
9
+ export declare const ARCHIVE_IO_CHUNK_BYTES: number;
10
+ export declare const LEGACY_BUFFER_LIMIT_BYTES: number;
11
+ export declare const MAX_FRAME_METADATA_BYTES: number;
12
+ export type ByteStream = AsyncIterable<Buffer>;
13
+ export type ByteDigest = {
14
+ bytes: number;
15
+ sha256Hex: string;
16
+ sha256Base64: string;
17
+ };
18
+ /**
19
+ * Applies a token-bucket style ceiling to a byte stream. One normal archive
20
+ * I/O chunk may leave immediately; sustained transfer then follows the chosen
21
+ * rate. The source is pulled only after the delay, so provider/file
22
+ * backpressure propagates all the way upstream instead of buffering ahead.
23
+ */
24
+ export declare function paceByteStream(chunks: ByteStream, bytesPerSecond: number, options?: {
25
+ signal?: AbortSignal;
26
+ burstBytes?: number;
27
+ now?: () => number;
28
+ sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
29
+ }): ByteStream;
30
+ /** A connected provider that makes no byte progress is not allowed to hold a
31
+ * queue lease, index build, or scratch reservation forever. This is an idle
32
+ * interval, reset after every yielded chunk, rather than a total-size limit. */
33
+ export declare const DEFAULT_REMOTE_READ_IDLE_MS = 60000;
34
+ /**
35
+ * Opens and consumes a provider stream under one progress watchdog.
36
+ *
37
+ * The derived signal reaches the control-plane request, provider fetch, and
38
+ * body iterator. Promise.race is still intentional: an injected or broken
39
+ * iterator may ignore AbortSignal, and its caller must nevertheless settle.
40
+ */
41
+ export declare function byteStreamWithIdleTimeout(openStream: (signal: AbortSignal) => Promise<ByteStream>, options?: {
42
+ signal?: AbortSignal;
43
+ idleMs?: number;
44
+ }): ByteStream;
45
+ /** Positional reads let one open descriptor be hashed and then uploaded. */
46
+ export declare function fileHandleChunks(handle: FileHandle, options?: {
47
+ start?: number;
48
+ end?: number;
49
+ chunkBytes?: number;
50
+ signal?: AbortSignal;
51
+ }): AsyncGenerator<Buffer>;
52
+ /** Opens a file for the lifetime of the iterator and always closes it. */
53
+ export declare function fileChunks(path: string, options?: {
54
+ start?: number;
55
+ end?: number;
56
+ chunkBytes?: number;
57
+ signal?: AbortSignal;
58
+ }): AsyncGenerator<Buffer>;
59
+ export declare function digestChunks(chunks: ByteStream, onProgress?: (bytes: number) => void): Promise<ByteDigest>;
60
+ export declare function digestFile(path: string, signal?: AbortSignal): Promise<ByteDigest>;
61
+ /**
62
+ * Writes an iterator without a stream-wide Buffer. The caller chooses the
63
+ * partial pathname and owns rename/cleanup; fsync makes the later rename a
64
+ * durability boundary instead of merely a namespace change.
65
+ */
66
+ export declare function writeChunksToFile(path: string, chunks: ByteStream, options?: {
67
+ mode?: number;
68
+ signal?: AbortSignal;
69
+ exclusive?: boolean;
70
+ sync?: boolean;
71
+ expectedBytes?: number;
72
+ onProgress?: (bytes: number) => void;
73
+ }): Promise<ByteDigest>;
74
+ /** A compatibility door for small injected/test clients; it refuses large files. */
75
+ export declare function collectChunks(chunks: ByteStream, limitBytes?: number, onProgress?: (bytes: number) => void): Promise<Buffer>;
76
+ /** Converts a successful fetch body to the same iterator used for files. */
77
+ export declare function responseBodyChunks(response: Response): AsyncGenerator<Buffer>;
78
+ /** Node fetch needs duplex=half when an iterator is used as a request body. */
79
+ export declare function requestBody(chunks: ByteStream): Readable;
80
+ export declare function concatStreams(streams: Iterable<ByteStream>): AsyncGenerator<Buffer>;
81
+ export declare function oneChunk(chunk: Uint8Array): AsyncGenerator<Buffer>;
82
+ /** Bounded zero-copy views over an already-buffered compatibility object. */
83
+ export declare function bufferChunks(buffer: Buffer, chunkBytes?: number): AsyncGenerator<Buffer>;
84
+ /** Updates a digest while leaving the byte stream otherwise untouched. */
85
+ export declare function tapDigest(chunks: ByteStream, hash: ReturnType<typeof createHash>, onBytes?: (bytes: number) => void): AsyncGenerator<Buffer>;
86
+ /**
87
+ * Removes the managed VLA1 envelope frame while downloading. Only the small,
88
+ * explicitly bounded JSON header is retained; ciphertext is yielded as soon
89
+ * as the header ends. Legacy/plain objects pass through byte-for-byte.
90
+ */
91
+ export declare function unframeChunks(chunks: ByteStream, magic?: Buffer<ArrayBuffer>, maxMetadataBytes?: number): AsyncGenerator<Buffer>;
@@ -0,0 +1,385 @@
1
+ import { createHash } from "node:crypto";
2
+ import { open } from "node:fs/promises";
3
+ import { Readable } from "node:stream";
4
+ import { fail, SealkeepError } from "./errors.js";
5
+ /**
6
+ * One deliberately small I/O window for archive copies. Providers may keep a
7
+ * part of their own (normally 8 MiB), but reading a multi-gigabyte file never
8
+ * creates a buffer proportional to the file.
9
+ */
10
+ export const ARCHIVE_IO_CHUNK_BYTES = 1024 * 1024;
11
+ export const LEGACY_BUFFER_LIMIT_BYTES = 64 * 1024 * 1024;
12
+ export const MAX_FRAME_METADATA_BYTES = 16 * 1024 * 1024;
13
+ /**
14
+ * Applies a token-bucket style ceiling to a byte stream. One normal archive
15
+ * I/O chunk may leave immediately; sustained transfer then follows the chosen
16
+ * rate. The source is pulled only after the delay, so provider/file
17
+ * backpressure propagates all the way upstream instead of buffering ahead.
18
+ */
19
+ export function paceByteStream(chunks, bytesPerSecond, options = {}) {
20
+ if (!Number.isFinite(bytesPerSecond) || bytesPerSecond <= 0)
21
+ throw new Error("Upload byte rate must be positive");
22
+ const burst = Math.max(0, options.burstBytes ?? ARCHIVE_IO_CHUNK_BYTES);
23
+ const now = options.now ?? Date.now;
24
+ const sleep = options.sleep ?? (async (ms, signal) => {
25
+ signal?.throwIfAborted();
26
+ await new Promise((resolve, reject) => {
27
+ let timer;
28
+ const aborted = () => {
29
+ if (timer)
30
+ clearTimeout(timer);
31
+ signal?.removeEventListener("abort", aborted);
32
+ reject(abortError(signal));
33
+ };
34
+ timer = setTimeout(() => { signal?.removeEventListener("abort", aborted); resolve(); }, ms);
35
+ signal?.addEventListener("abort", aborted, { once: true });
36
+ });
37
+ });
38
+ return {
39
+ async *[Symbol.asyncIterator]() {
40
+ const started = now();
41
+ let sent = 0;
42
+ for await (const chunk of chunks) {
43
+ options.signal?.throwIfAborted();
44
+ const charged = Math.max(0, sent + chunk.length - burst);
45
+ const dueAt = started + charged / bytesPerSecond * 1_000;
46
+ const delay = dueAt - now();
47
+ if (delay > 0)
48
+ await sleep(delay, options.signal);
49
+ yield chunk;
50
+ sent += chunk.length;
51
+ }
52
+ },
53
+ };
54
+ }
55
+ /** A connected provider that makes no byte progress is not allowed to hold a
56
+ * queue lease, index build, or scratch reservation forever. This is an idle
57
+ * interval, reset after every yielded chunk, rather than a total-size limit. */
58
+ export const DEFAULT_REMOTE_READ_IDLE_MS = 60_000;
59
+ function remoteReadTimeout(idleMs) {
60
+ return new SealkeepError("storage_read_timeout", `Stored archive reading made no progress for ${Math.max(1, Math.round(idleMs / 1_000))} seconds. The partial copy was stopped and can be retried safely.`, { retryable: true, idleMs });
61
+ }
62
+ /**
63
+ * Opens and consumes a provider stream under one progress watchdog.
64
+ *
65
+ * The derived signal reaches the control-plane request, provider fetch, and
66
+ * body iterator. Promise.race is still intentional: an injected or broken
67
+ * iterator may ignore AbortSignal, and its caller must nevertheless settle.
68
+ */
69
+ export function byteStreamWithIdleTimeout(openStream, options = {}) {
70
+ const idleMs = options.idleMs ?? DEFAULT_REMOTE_READ_IDLE_MS;
71
+ if (!Number.isFinite(idleMs) || idleMs <= 0)
72
+ throw new Error("Remote archive idle timeout must be positive");
73
+ return {
74
+ async *[Symbol.asyncIterator]() {
75
+ const controller = new AbortController();
76
+ const abortFromParent = () => controller.abort(options.signal?.reason ?? new DOMException("The archive transfer was cancelled", "AbortError"));
77
+ if (options.signal?.aborted)
78
+ abortFromParent();
79
+ else
80
+ options.signal?.addEventListener("abort", abortFromParent, { once: true });
81
+ let timer;
82
+ let iterator;
83
+ let complete = false;
84
+ let idleDeadline = Date.now() + idleMs;
85
+ const waitForProgress = async (work) => {
86
+ if (controller.signal.aborted)
87
+ throw abortError(controller.signal);
88
+ let onAbort;
89
+ timer = setTimeout(() => controller.abort(remoteReadTimeout(idleMs)), Math.max(1, idleDeadline - Date.now()));
90
+ const aborted = new Promise((_resolve, reject) => {
91
+ onAbort = () => reject(abortError(controller.signal));
92
+ controller.signal.addEventListener("abort", onAbort, { once: true });
93
+ });
94
+ try {
95
+ return await Promise.race([work, aborted]);
96
+ }
97
+ finally {
98
+ if (timer)
99
+ clearTimeout(timer);
100
+ timer = undefined;
101
+ if (onAbort)
102
+ controller.signal.removeEventListener("abort", onAbort);
103
+ }
104
+ };
105
+ try {
106
+ const source = await waitForProgress(openStream(controller.signal));
107
+ iterator = source[Symbol.asyncIterator]();
108
+ for (;;) {
109
+ const item = await waitForProgress(iterator.next());
110
+ if (item.done) {
111
+ complete = true;
112
+ break;
113
+ }
114
+ const chunk = Buffer.isBuffer(item.value) ? item.value : Buffer.from(item.value);
115
+ // Empty iterator churn is not byte progress and must not keep a job
116
+ // alive forever. A positive chunk resets the budget only when the
117
+ // consumer asks for the next one, so its own processing time is not
118
+ // charged to the provider.
119
+ if (chunk.length === 0)
120
+ continue;
121
+ yield chunk;
122
+ idleDeadline = Date.now() + idleMs;
123
+ }
124
+ }
125
+ finally {
126
+ if (timer)
127
+ clearTimeout(timer);
128
+ options.signal?.removeEventListener("abort", abortFromParent);
129
+ if (!complete) {
130
+ if (!controller.signal.aborted)
131
+ controller.abort(new DOMException("Archive stream consumer stopped", "AbortError"));
132
+ if (iterator?.return) {
133
+ try {
134
+ void iterator.return().catch(() => undefined);
135
+ }
136
+ catch { /* a malformed custom iterator cannot replace the primary failure */ }
137
+ }
138
+ }
139
+ }
140
+ },
141
+ };
142
+ }
143
+ function abortError(signal) {
144
+ if (!signal?.aborted)
145
+ return null;
146
+ if (signal.reason instanceof Error)
147
+ return signal.reason;
148
+ const error = new Error("The archive transfer was cancelled");
149
+ error.name = "AbortError";
150
+ return error;
151
+ }
152
+ /** Positional reads let one open descriptor be hashed and then uploaded. */
153
+ export async function* fileHandleChunks(handle, options = {}) {
154
+ const chunkBytes = options.chunkBytes ?? ARCHIVE_IO_CHUNK_BYTES;
155
+ if (!Number.isInteger(chunkBytes) || chunkBytes <= 0)
156
+ throw new Error("Archive stream chunk size must be a positive integer");
157
+ let position = options.start ?? 0;
158
+ const end = options.end ?? Number.POSITIVE_INFINITY;
159
+ while (position < end) {
160
+ const cancelled = abortError(options.signal);
161
+ if (cancelled)
162
+ throw cancelled;
163
+ const wanted = Math.min(chunkBytes, end - position);
164
+ const buffer = Buffer.allocUnsafe(wanted);
165
+ const { bytesRead } = await handle.read(buffer, 0, wanted, position);
166
+ if (bytesRead === 0)
167
+ break;
168
+ position += bytesRead;
169
+ yield bytesRead === buffer.length ? buffer : buffer.subarray(0, bytesRead);
170
+ }
171
+ }
172
+ /** Opens a file for the lifetime of the iterator and always closes it. */
173
+ export async function* fileChunks(path, options = {}) {
174
+ const handle = await open(path, "r");
175
+ try {
176
+ yield* fileHandleChunks(handle, options);
177
+ }
178
+ finally {
179
+ await handle.close();
180
+ }
181
+ }
182
+ export async function digestChunks(chunks, onProgress) {
183
+ const hash = createHash("sha256");
184
+ let bytes = 0;
185
+ for await (const value of chunks) {
186
+ const chunk = value;
187
+ hash.update(chunk);
188
+ bytes += chunk.length;
189
+ onProgress?.(bytes);
190
+ }
191
+ const digest = hash.digest();
192
+ return { bytes, sha256Hex: digest.toString("hex"), sha256Base64: digest.toString("base64") };
193
+ }
194
+ export async function digestFile(path, signal) {
195
+ return digestChunks(fileChunks(path, { signal }));
196
+ }
197
+ /**
198
+ * Writes an iterator without a stream-wide Buffer. The caller chooses the
199
+ * partial pathname and owns rename/cleanup; fsync makes the later rename a
200
+ * durability boundary instead of merely a namespace change.
201
+ */
202
+ export async function writeChunksToFile(path, chunks, options = {}) {
203
+ if (options.expectedBytes !== undefined
204
+ && (!Number.isSafeInteger(options.expectedBytes) || options.expectedBytes < 0)) {
205
+ fail("ciphertext_integrity_failed", "The archive record has no valid exact ciphertext length, so no staging file was opened.", { expectedBytes: options.expectedBytes });
206
+ }
207
+ const handle = await open(path, options.exclusive === false ? "w" : "wx", options.mode ?? 0o600);
208
+ const hash = createHash("sha256");
209
+ let bytes = 0;
210
+ const iterator = chunks[Symbol.asyncIterator]();
211
+ let complete = false;
212
+ const next = async () => {
213
+ const cancelled = abortError(options.signal);
214
+ if (cancelled)
215
+ throw cancelled;
216
+ if (!options.signal)
217
+ return iterator.next();
218
+ let onAbort;
219
+ const aborted = new Promise((_resolve, reject) => {
220
+ onAbort = () => reject(abortError(options.signal));
221
+ options.signal.addEventListener("abort", onAbort, { once: true });
222
+ });
223
+ try {
224
+ return await Promise.race([iterator.next(), aborted]);
225
+ }
226
+ finally {
227
+ if (onAbort)
228
+ options.signal.removeEventListener("abort", onAbort);
229
+ }
230
+ };
231
+ try {
232
+ for (;;) {
233
+ const item = await next();
234
+ if (item.done) {
235
+ complete = true;
236
+ break;
237
+ }
238
+ const cancelled = abortError(options.signal);
239
+ if (cancelled)
240
+ throw cancelled;
241
+ const chunk = item.value;
242
+ if (options.expectedBytes !== undefined && chunk.length > options.expectedBytes - bytes) {
243
+ fail("ciphertext_integrity_failed", `The archive source sent more than its recorded ${options.expectedBytes} ciphertext bytes; the excess was refused before it reached disk.`, { expectedBytes: options.expectedBytes, receivedAtLeast: bytes + chunk.length });
244
+ }
245
+ hash.update(chunk);
246
+ bytes += chunk.length;
247
+ options.onProgress?.(bytes);
248
+ let written = 0;
249
+ while (written < chunk.length) {
250
+ const result = await handle.write(chunk, written, chunk.length - written, null);
251
+ if (result.bytesWritten <= 0)
252
+ throw new Error(`Could not finish writing archive staging file ${path}`);
253
+ written += result.bytesWritten;
254
+ }
255
+ }
256
+ if (options.expectedBytes !== undefined && bytes !== options.expectedBytes) {
257
+ fail("ciphertext_integrity_failed", `The archive source ended after ${bytes} ciphertext bytes; its record requires ${options.expectedBytes}.`, { expectedBytes: options.expectedBytes, receivedBytes: bytes });
258
+ }
259
+ if (options.sync !== false)
260
+ await handle.sync();
261
+ }
262
+ finally {
263
+ if (!complete && typeof iterator.return === "function") {
264
+ // Do not let an iterator that ignored AbortSignal strand the caller too.
265
+ // Production HTTP streams receive the same signal and cancel promptly;
266
+ // this best-effort return releases cooperative injected/custom sources.
267
+ void iterator.return().catch(() => undefined);
268
+ }
269
+ await handle.close();
270
+ }
271
+ const digest = hash.digest();
272
+ return { bytes, sha256Hex: digest.toString("hex"), sha256Base64: digest.toString("base64") };
273
+ }
274
+ /** A compatibility door for small injected/test clients; it refuses large files. */
275
+ export async function collectChunks(chunks, limitBytes = LEGACY_BUFFER_LIMIT_BYTES, onProgress) {
276
+ const held = [];
277
+ let bytes = 0;
278
+ for await (const value of chunks) {
279
+ const chunk = value;
280
+ bytes += chunk.length;
281
+ onProgress?.(bytes);
282
+ if (bytes > limitBytes) {
283
+ throw new Error(`This client only accepts buffered transfers, and the archive exceeds the ${limitBytes}-byte safety limit`);
284
+ }
285
+ held.push(chunk);
286
+ }
287
+ return Buffer.concat(held, bytes);
288
+ }
289
+ /** Converts a successful fetch body to the same iterator used for files. */
290
+ export async function* responseBodyChunks(response) {
291
+ if (!response.body)
292
+ return;
293
+ for await (const value of response.body) {
294
+ yield Buffer.from(value.buffer, value.byteOffset, value.byteLength);
295
+ }
296
+ }
297
+ /** Node fetch needs duplex=half when an iterator is used as a request body. */
298
+ export function requestBody(chunks) {
299
+ return Readable.from(chunks);
300
+ }
301
+ export async function* concatStreams(streams) {
302
+ for (const stream of streams)
303
+ yield* stream;
304
+ }
305
+ export async function* oneChunk(chunk) {
306
+ yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
307
+ }
308
+ /** Bounded zero-copy views over an already-buffered compatibility object. */
309
+ export async function* bufferChunks(buffer, chunkBytes = ARCHIVE_IO_CHUNK_BYTES) {
310
+ if (!Number.isInteger(chunkBytes) || chunkBytes <= 0)
311
+ throw new Error("Buffer chunk size must be a positive integer");
312
+ for (let at = 0; at < buffer.length; at += chunkBytes)
313
+ yield buffer.subarray(at, Math.min(buffer.length, at + chunkBytes));
314
+ }
315
+ /** Updates a digest while leaving the byte stream otherwise untouched. */
316
+ export async function* tapDigest(chunks, hash, onBytes) {
317
+ for await (const value of chunks) {
318
+ const chunk = value;
319
+ hash.update(chunk);
320
+ onBytes?.(chunk.length);
321
+ yield chunk;
322
+ }
323
+ }
324
+ /**
325
+ * Removes the managed VLA1 envelope frame while downloading. Only the small,
326
+ * explicitly bounded JSON header is retained; ciphertext is yielded as soon
327
+ * as the header ends. Legacy/plain objects pass through byte-for-byte.
328
+ */
329
+ export async function* unframeChunks(chunks, magic = Buffer.from("VLA1", "ascii"), maxMetadataBytes = MAX_FRAME_METADATA_BYTES) {
330
+ let decided = false;
331
+ let framed = false;
332
+ let metadataBytes = null;
333
+ let held = [];
334
+ let heldBytes = 0;
335
+ for await (const value of chunks) {
336
+ const chunk = value;
337
+ if (decided) {
338
+ yield chunk;
339
+ continue;
340
+ }
341
+ held.push(chunk);
342
+ heldBytes += chunk.length;
343
+ if (heldBytes < magic.length)
344
+ continue;
345
+ let buffered = Buffer.concat(held, heldBytes);
346
+ if (!framed) {
347
+ if (!buffered.subarray(0, magic.length).equals(magic)) {
348
+ decided = true;
349
+ yield buffered;
350
+ held = [];
351
+ heldBytes = 0;
352
+ continue;
353
+ }
354
+ framed = true;
355
+ }
356
+ if (buffered.length < magic.length + 4)
357
+ continue;
358
+ metadataBytes ??= buffered.readUInt32BE(magic.length);
359
+ if (metadataBytes > maxMetadataBytes) {
360
+ throw new Error(`Managed archive metadata is ${metadataBytes} bytes; the safe limit is ${maxMetadataBytes}`);
361
+ }
362
+ const bodyStart = magic.length + 4 + metadataBytes;
363
+ if (buffered.length < bodyStart)
364
+ continue;
365
+ try {
366
+ JSON.parse(buffered.subarray(magic.length + 4, bodyStart).toString("utf8"));
367
+ }
368
+ catch {
369
+ throw new Error("Managed archive header is not readable JSON");
370
+ }
371
+ decided = true;
372
+ held = [];
373
+ heldBytes = 0;
374
+ buffered = buffered.subarray(bodyStart);
375
+ if (buffered.length > 0)
376
+ yield buffered;
377
+ }
378
+ if (!decided && heldBytes > 0) {
379
+ const buffered = Buffer.concat(held, heldBytes);
380
+ if (!framed)
381
+ yield buffered;
382
+ else
383
+ throw new Error("Managed archive header is truncated");
384
+ }
385
+ }
@@ -1,5 +1,6 @@
1
- import type { ArchiveRecordV2, RemoteObject } from "./types.js";
2
- import { type ChunkHeader } from "../packages/vaultline-crypto/src/index.js";
1
+ import { type ArchiveRecordV2, type RemoteObject } from "./types.js";
2
+ import { type ChunkHeader } from "../packages/sealkeep-crypto/src/index.js";
3
+ import type { Recipient } from "../packages/sealkeep-crypto/src/index.js";
3
4
  /**
4
5
  * Remote layout v2: a session's archive is a FOLDER of chunk objects, not one
5
6
  * blob. Paul's shape, and the natural completion of the gzip-chunk format —
@@ -10,7 +11,7 @@ import { type ChunkHeader } from "../packages/vaultline-crypto/src/index.js";
10
11
  *
11
12
  * <project>/<YYYY-MM-DD>/<session>.<id8>/
12
13
  * chunk-000000 … chunk-NNNNNN one 4 MB sealed chunk per object
13
- * envelope.vlmeta the wrapped-keys envelope
14
+ * envelope.skmeta the wrapped-keys envelope
14
15
  *
15
16
  * What the folder buys:
16
17
  * - A PUT per chunk needs no multipart anything: length known, checksum
@@ -38,19 +39,36 @@ import { type ChunkHeader } from "../packages/vaultline-crypto/src/index.js";
38
39
  * abandons the folder and restarts under a fresh key.
39
40
  */
40
41
  export type ChunkStoreClient = {
41
- putObject(key: string, body: Buffer): Promise<{
42
+ putObject(key: string, body: Buffer, options?: {
43
+ signal?: AbortSignal;
44
+ }): Promise<{
42
45
  checksum: string;
43
46
  }>;
44
- headObject(key: string): Promise<{
47
+ headObject(key: string, options?: {
48
+ signal?: AbortSignal;
49
+ }): Promise<{
45
50
  exists: boolean;
46
51
  bytes: number;
47
52
  checksum?: string;
48
53
  }>;
49
- getObject(key: string): Promise<Buffer>;
54
+ getObject(key: string, options?: {
55
+ signal?: AbortSignal;
56
+ }): Promise<Buffer>;
57
+ /** Optional production streaming verbs used by large index snapshots. */
58
+ uploadStream?(key: string, chunks: AsyncIterable<Buffer>, options?: {
59
+ signal?: AbortSignal;
60
+ bytes?: number;
61
+ }): Promise<{
62
+ remoteChecksum: string | null;
63
+ bytes: number;
64
+ }>;
65
+ getObjectStream?(key: string, options?: {
66
+ signal?: AbortSignal;
67
+ }): Promise<AsyncIterable<Buffer>>;
50
68
  deleteObject(key: string): Promise<void>;
51
69
  };
52
70
  export declare const chunkObjectName: (index: number) => string;
53
- export declare const ENVELOPE_OBJECT = "envelope.vlmeta";
71
+ export declare const ENVELOPE_OBJECT = "envelope.skmeta";
54
72
  export type RemoteNaming = "readable" | "hashed";
55
73
  /**
56
74
  * The key the hashed naming mode HMACs segments under, derived from the
@@ -115,9 +133,16 @@ export type ChunkSealOutcome = {
115
133
  export declare function sealToChunkFolder(dataDir: string, sourcePath: string, rawPhrase: string, agent: string, options?: {
116
134
  client?: ChunkStoreClient;
117
135
  project?: string | null;
136
+ projectKey?: string | null;
137
+ teamRecipients?: Recipient[];
118
138
  resume?: boolean;
119
139
  now?: number;
120
140
  targetId?: string;
141
+ signal?: AbortSignal;
142
+ chunkBytes?: number;
143
+ maxBytesPerSecond?: number;
144
+ /** Internal cooperative scheduler used by unattended/background sealing. */
145
+ onYield?: () => void | Promise<void>;
121
146
  }): Promise<ChunkSealOutcome>;
122
147
  /**
123
148
  * Seals the readable identity for the sidecar. Its own random nonce — chunk
@@ -128,13 +153,21 @@ export declare function sealToChunkFolder(dataDir: string, sourcePath: string, r
128
153
  export type FolderIdentity = {
129
154
  v: 1;
130
155
  project: string | null;
156
+ projectKey?: string;
157
+ projectScope?: string;
131
158
  sourcePath: string;
132
159
  agent: string;
133
160
  createdAt: string;
134
161
  };
135
162
  /** Opens a sidecar's identity with the unwrapped archive key. */
136
163
  export declare function openIdentity(archiveKey: Buffer, archiveId: string, blob: Buffer): FolderIdentity;
137
- /** Fetches a chunked remote back into one ciphertext buffer, hash-checked per chunk. */
164
+ /** Fetches a chunked remote one bounded object at a time, hash-checking before yield. */
165
+ export declare function fetchChunkFolderStream(client: ChunkStoreClient, remote: RemoteObject, expected: {
166
+ chunks: ChunkHeader[];
167
+ }, options?: {
168
+ signal?: AbortSignal;
169
+ }): AsyncGenerator<Buffer>;
170
+ /** Compatibility API for callers that explicitly need the complete Buffer. */
138
171
  export declare function fetchChunkFolder(client: ChunkStoreClient, remote: RemoteObject, expected: {
139
172
  chunks: ChunkHeader[];
140
173
  }): Promise<Buffer>;