sealkeep 0.8.1 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. package/ARCHITECTURE.md +163 -14
  2. package/CHANGELOG.md +244 -1
  3. package/CONTROL_PLANE.md +2 -2
  4. package/LICENSE +1 -1
  5. package/README.md +108 -22
  6. package/THIRD_PARTY.md +2 -2
  7. package/THREAT_MODEL.md +23 -4
  8. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/chunk-access.d.ts +26 -4
  9. package/dist/packages/sealkeep-crypto/src/chunk-access.js +219 -0
  10. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/envelope.d.ts +11 -1
  11. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/envelope.js +58 -21
  12. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/format.d.ts +2 -2
  13. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/index.d.ts +1 -0
  14. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/index.js +1 -0
  15. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/recipients.js +4 -4
  16. package/dist/packages/sealkeep-crypto/src/squeeze.d.ts +6 -0
  17. package/dist/packages/sealkeep-crypto/src/squeeze.js +39 -0
  18. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/stream.d.ts +36 -3
  19. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/stream.js +196 -37
  20. package/dist/site/index.html +1808 -1904
  21. package/dist/site/llms.txt +67 -0
  22. package/dist/site/trust/architecture-data-flow.html +53 -0
  23. package/dist/site/trust/audit-roadmap.html +37 -0
  24. package/dist/site/trust/deployment-responsibility.html +11 -0
  25. package/dist/site/trust/dpa-sample.html +30 -0
  26. package/dist/site/trust/release-provenance.html +21 -0
  27. package/dist/site/trust/subprocessors.html +15 -0
  28. package/dist/site/trust/threat-model.html +34 -0
  29. package/dist/site/trust/toms.html +41 -0
  30. package/dist/site/trust-document.css +32 -0
  31. package/dist/site/trust.html +73 -0
  32. package/dist/site/visual/assets/index-8Nxnkc7o.js +207 -0
  33. package/dist/site/visual/assets/index-oxLy1bpy.css +1 -0
  34. package/dist/site/visual/index.html +18 -0
  35. package/dist/site.zip +0 -0
  36. package/dist/src/activity.d.ts +9 -0
  37. package/dist/src/activity.js +90 -1
  38. package/dist/src/adapters.d.ts +175 -5
  39. package/dist/src/adapters.js +961 -73
  40. package/dist/src/agent-context.d.ts +135 -0
  41. package/dist/src/agent-context.js +1059 -0
  42. package/dist/src/archive-copies.d.ts +47 -0
  43. package/dist/src/archive-copies.js +179 -0
  44. package/dist/src/audit.d.ts +1 -1
  45. package/dist/src/audit.js +29 -4
  46. package/dist/src/autopilot.d.ts +52 -7
  47. package/dist/src/autopilot.js +143 -25
  48. package/dist/src/background-bandwidth.d.ts +46 -0
  49. package/dist/src/background-bandwidth.js +301 -0
  50. package/dist/src/background-cpu.d.ts +82 -0
  51. package/dist/src/background-cpu.js +212 -0
  52. package/dist/src/background-worker-error.d.ts +12 -0
  53. package/dist/src/background-worker-error.js +18 -0
  54. package/dist/src/branding.d.ts +24 -1
  55. package/dist/src/branding.js +31 -1
  56. package/dist/src/bridge.d.ts +233 -0
  57. package/dist/src/bridge.js +604 -0
  58. package/dist/src/byte-stream.d.ts +91 -0
  59. package/dist/src/byte-stream.js +385 -0
  60. package/dist/src/chunk-store.d.ts +41 -8
  61. package/dist/src/chunk-store.js +161 -65
  62. package/dist/src/cli.js +1746 -166
  63. package/dist/src/cloud.d.ts +841 -31
  64. package/dist/src/cloud.js +3196 -277
  65. package/dist/src/context-background.d.ts +37 -0
  66. package/dist/src/context-background.js +309 -0
  67. package/dist/src/context-drain-child.d.ts +1 -0
  68. package/dist/src/context-drain-child.js +98 -0
  69. package/dist/src/context-reader.d.ts +118 -0
  70. package/dist/src/context-reader.js +447 -0
  71. package/dist/src/control-plane/auth.d.ts +32 -4
  72. package/dist/src/control-plane/auth.js +85 -24
  73. package/dist/src/control-plane/server.js +19 -6
  74. package/dist/src/control-plane.d.ts +17 -1
  75. package/dist/src/control-plane.js +32 -6
  76. package/dist/src/crypto.d.ts +1 -1
  77. package/dist/src/crypto.js +5 -5
  78. package/dist/src/daemon-lease.d.ts +70 -0
  79. package/dist/src/daemon-lease.js +420 -0
  80. package/dist/src/daemon.d.ts +94 -1
  81. package/dist/src/daemon.js +1082 -105
  82. package/dist/src/darwin-service-policy.d.ts +41 -0
  83. package/dist/src/darwin-service-policy.js +60 -0
  84. package/dist/src/dashboard-cli.js +15 -15
  85. package/dist/src/device-authorization.d.ts +37 -0
  86. package/dist/src/device-authorization.js +199 -0
  87. package/dist/src/device-enrollment.d.ts +91 -0
  88. package/dist/src/device-enrollment.js +349 -0
  89. package/dist/src/disk.d.ts +17 -12
  90. package/dist/src/disk.js +43 -17
  91. package/dist/src/doctor.d.ts +35 -1
  92. package/dist/src/doctor.js +316 -41
  93. package/dist/src/durable-ticket-lock.d.ts +24 -0
  94. package/dist/src/durable-ticket-lock.js +232 -0
  95. package/dist/src/enroll.d.ts +1 -1
  96. package/dist/src/enroll.js +13 -7
  97. package/dist/src/env.d.ts +10 -1
  98. package/dist/src/env.js +11 -3
  99. package/dist/src/errors.d.ts +8 -8
  100. package/dist/src/errors.js +6 -6
  101. package/dist/src/flush.d.ts +12 -0
  102. package/dist/src/flush.js +37 -0
  103. package/dist/src/heartbeat.d.ts +86 -12
  104. package/dist/src/heartbeat.js +415 -29
  105. package/dist/src/index-background-watchdog.d.ts +1 -0
  106. package/dist/src/index-background-watchdog.js +94 -0
  107. package/dist/src/index-background-work.d.ts +21 -0
  108. package/dist/src/index-background-work.js +25 -0
  109. package/dist/src/index-background.d.ts +64 -0
  110. package/dist/src/index-background.js +394 -0
  111. package/dist/src/index-build-child.d.ts +1 -0
  112. package/dist/src/index-build-child.js +109 -0
  113. package/dist/src/index-manifest.d.ts +52 -0
  114. package/dist/src/index-manifest.js +444 -0
  115. package/dist/src/index-publication-proof.d.ts +84 -0
  116. package/dist/src/index-publication-proof.js +380 -0
  117. package/dist/src/index-publication-state.d.ts +149 -0
  118. package/dist/src/index-publication-state.js +696 -0
  119. package/dist/src/index-publication-verifier.d.ts +89 -0
  120. package/dist/src/index-publication-verifier.js +341 -0
  121. package/dist/src/index-publish.d.ts +62 -0
  122. package/dist/src/index-publish.js +540 -0
  123. package/dist/src/index-scratch-cleanup.d.ts +19 -0
  124. package/dist/src/index-scratch-cleanup.js +166 -0
  125. package/dist/src/index-segment-types.d.ts +132 -0
  126. package/dist/src/index-segment-types.js +21 -0
  127. package/dist/src/index-segments.d.ts +9 -0
  128. package/dist/src/index-segments.js +516 -0
  129. package/dist/src/index-store.d.ts +123 -0
  130. package/dist/src/index-store.js +495 -0
  131. package/dist/src/index-sync.d.ts +91 -15
  132. package/dist/src/index-sync.js +286 -53
  133. package/dist/src/index-upgrade-publication.d.ts +30 -0
  134. package/dist/src/index-upgrade-publication.js +179 -0
  135. package/dist/src/integration-manager.d.ts +32 -0
  136. package/dist/src/integration-manager.js +394 -0
  137. package/dist/src/leakscan.js +1 -1
  138. package/dist/src/local-api.d.ts +56 -15
  139. package/dist/src/local-api.js +4979 -541
  140. package/dist/src/machine-settings.d.ts +51 -0
  141. package/dist/src/machine-settings.js +166 -0
  142. package/dist/src/managed-chunks.d.ts +5 -2
  143. package/dist/src/managed-chunks.js +14 -14
  144. package/dist/src/mcp-install.d.ts +11 -9
  145. package/dist/src/mcp-install.js +73 -28
  146. package/dist/src/mcp-workspace.d.ts +18 -0
  147. package/dist/src/mcp-workspace.js +50 -0
  148. package/dist/src/mcp.js +294 -25
  149. package/dist/src/migrate.d.ts +1 -0
  150. package/dist/src/migrate.js +33 -5
  151. package/dist/src/notify.d.ts +1 -1
  152. package/dist/src/notify.js +5 -5
  153. package/dist/src/offload.d.ts +201 -14
  154. package/dist/src/offload.js +1848 -140
  155. package/dist/src/onboarding.d.ts +8 -1
  156. package/dist/src/onboarding.js +4 -4
  157. package/dist/src/packages.d.ts +2 -2
  158. package/dist/src/packages.js +10 -2
  159. package/dist/src/passkey.d.ts +0 -1
  160. package/dist/src/passkey.js +2 -7
  161. package/dist/src/password-lock.d.ts +2 -2
  162. package/dist/src/password-lock.js +6 -6
  163. package/dist/src/paths.d.ts +2 -0
  164. package/dist/src/paths.js +2 -0
  165. package/dist/src/presence.d.ts +86 -0
  166. package/dist/src/presence.js +240 -0
  167. package/dist/src/progress-deadline.d.ts +21 -0
  168. package/dist/src/progress-deadline.js +91 -0
  169. package/dist/src/project-repair.d.ts +55 -0
  170. package/dist/src/project-repair.js +131 -0
  171. package/dist/src/providers/gcs.d.ts +28 -7
  172. package/dist/src/providers/gcs.js +35 -24
  173. package/dist/src/providers/gdrive.d.ts +71 -8
  174. package/dist/src/providers/gdrive.js +223 -50
  175. package/dist/src/providers/index.d.ts +11 -3
  176. package/dist/src/providers/index.js +30 -10
  177. package/dist/src/providers/s3.d.ts +30 -8
  178. package/dist/src/providers/s3.js +41 -30
  179. package/dist/src/providers/safe-storage-fetch.d.ts +12 -0
  180. package/dist/src/providers/safe-storage-fetch.js +72 -0
  181. package/dist/src/queue.d.ts +136 -19
  182. package/dist/src/queue.js +862 -96
  183. package/dist/src/reclaim-transaction.d.ts +156 -0
  184. package/dist/src/reclaim-transaction.js +1027 -0
  185. package/dist/src/recovery-codes.d.ts +32 -0
  186. package/dist/src/recovery-codes.js +338 -0
  187. package/dist/src/recovery.js +12 -9
  188. package/dist/src/rehydrate.d.ts +25 -22
  189. package/dist/src/rehydrate.js +319 -23
  190. package/dist/src/restore.d.ts +57 -4
  191. package/dist/src/restore.js +272 -36
  192. package/dist/src/resume-stub.d.ts +92 -0
  193. package/dist/src/resume-stub.js +417 -0
  194. package/dist/src/retention.d.ts +98 -7
  195. package/dist/src/retention.js +1066 -62
  196. package/dist/src/rotate.js +3 -3
  197. package/dist/src/search.d.ts +566 -8
  198. package/dist/src/search.js +5897 -290
  199. package/dist/src/secrets.d.ts +51 -7
  200. package/dist/src/secrets.js +316 -24
  201. package/dist/src/service.d.ts +49 -11
  202. package/dist/src/service.js +776 -35
  203. package/dist/src/share.js +3 -3
  204. package/dist/src/shared-spaces.d.ts +98 -0
  205. package/dist/src/shared-spaces.js +214 -0
  206. package/dist/src/source-reader.d.ts +73 -0
  207. package/dist/src/source-reader.js +715 -0
  208. package/dist/src/spool.d.ts +1 -1
  209. package/dist/src/spool.js +1 -1
  210. package/dist/src/start-tui.js +2 -1
  211. package/dist/src/start.js +2 -2
  212. package/dist/src/storage-endpoint.d.ts +21 -0
  213. package/dist/src/storage-endpoint.js +122 -0
  214. package/dist/src/storage-setup.js +12 -12
  215. package/dist/src/storage-targets.d.ts +109 -6
  216. package/dist/src/storage-targets.js +975 -67
  217. package/dist/src/stream-to-cloud.d.ts +5 -1
  218. package/dist/src/stream-to-cloud.js +34 -14
  219. package/dist/src/sync-rules.d.ts +31 -6
  220. package/dist/src/sync-rules.js +153 -14
  221. package/dist/src/team-backfill-scheduling.d.ts +8 -0
  222. package/dist/src/team-backfill-scheduling.js +33 -0
  223. package/dist/src/team-backfill.d.ts +116 -0
  224. package/dist/src/team-backfill.js +1429 -0
  225. package/dist/src/team-index-cache.d.ts +16 -0
  226. package/dist/src/team-index-cache.js +152 -0
  227. package/dist/src/team-offboarding.d.ts +38 -0
  228. package/dist/src/team-offboarding.js +1043 -0
  229. package/dist/src/team-presence.d.ts +127 -0
  230. package/dist/src/team-presence.js +904 -0
  231. package/dist/src/team-publication-policy.d.ts +20 -0
  232. package/dist/src/team-publication-policy.js +140 -0
  233. package/dist/src/team-realtime.d.ts +68 -0
  234. package/dist/src/team-realtime.js +816 -0
  235. package/dist/src/team-source-facts-cache.d.ts +23 -0
  236. package/dist/src/team-source-facts-cache.js +255 -0
  237. package/dist/src/trash.d.ts +1 -1
  238. package/dist/src/trash.js +2 -2
  239. package/dist/src/tui.js +11 -12
  240. package/dist/src/types.d.ts +173 -7
  241. package/dist/src/types.js +20 -0
  242. package/dist/src/ui-server.d.ts +163 -35
  243. package/dist/src/ui-server.js +712 -72
  244. package/dist/src/ui.d.ts +1 -2
  245. package/dist/src/ui.js +1 -2
  246. package/dist/src/upload.d.ts +27 -0
  247. package/dist/src/upload.js +383 -43
  248. package/dist/src/vault.d.ts +235 -29
  249. package/dist/src/vault.js +1783 -190
  250. package/dist/src/watcher.d.ts +7 -1
  251. package/dist/src/watcher.js +198 -55
  252. package/dist/src/worker.d.ts +27 -3
  253. package/dist/src/worker.js +274 -55
  254. package/package.json +33 -12
  255. package/scripts/native-reboot-rehearsal.mjs +90 -0
  256. package/web/app.js +6032 -343
  257. package/web/bootstrap.js +17 -0
  258. package/web/index.html +255 -57
  259. package/web/rail.js +317 -40
  260. package/web/retention.html +2 -2
  261. package/web/rules-view.js +188 -16
  262. package/web/sessions-view.js +485 -62
  263. package/web/sessions.html +2 -2
  264. package/web/setup-api.js +152 -29
  265. package/web/setup-logic.js +68 -9
  266. package/web/setup.html +113 -44
  267. package/web/setup.js +604 -71
  268. package/web/style.css +513 -98
  269. package/dist/packages/vaultline-crypto/src/chunk-access.js +0 -93
  270. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/aead.d.ts +0 -0
  271. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/aead.js +0 -0
  272. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/format.js +0 -0
  273. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/recipients.d.ts +0 -0
  274. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/sha256-stream.d.ts +0 -0
  275. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/sha256-stream.js +0 -0
@@ -1,12 +1,56 @@
1
- import { mkdtemp, rm, stat, writeFile } from "node:fs/promises";
1
+ import { link as hardLink, lstat, mkdir, mkdtemp, open, readFile, readdir, realpath, rename, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
2
+ import { openForFlush } from "./flush.js";
3
+ import { constants as fsConstants } from "node:fs";
4
+ import { createHash, randomUUID } from "node:crypto";
2
5
  import { tmpdir } from "node:os";
3
- import { join } from "node:path";
4
- import { fail } from "./errors.js";
6
+ import { basename, dirname, join, resolve } from "node:path";
7
+ import { fail, SealkeepError } from "./errors.js";
5
8
  import { recordAudit } from "./audit.js";
6
9
  import { isV2 } from "./types.js";
7
- import { listArchives, readConfig, writeRecord } from "./vault.js";
10
+ import { listArchives, mutateArchiveRecord, readConfig, resolveDeltaChain } from "./vault.js";
8
11
  import { createActiveLease, uploadClientFromStore } from "./providers/index.js";
9
- import { sha256 } from "./crypto.js";
12
+ import { archiveCopies, verifiedCopyDestinationKey } from "./archive-copies.js";
13
+ import { byteStreamWithIdleTimeout, digestChunks, LEGACY_BUFFER_LIMIT_BYTES, oneChunk, writeChunksToFile } from "./byte-stream.js";
14
+ import { classifyDaemonProcessGeneration, daemonProcessIdentity } from "./daemon-lease.js";
15
+ import { acquireDurableTicketLock } from "./durable-ticket-lock.js";
16
+ import { paceBackgroundByteStream } from "./background-bandwidth.js";
17
+ import { validateNoFollowPath } from "./reclaim-transaction.js";
18
+ /** Resolve the immutable location recorded on a copy back to its account target. */
19
+ async function configuredTargetForRemote(dataDir, remote) {
20
+ const config = await readConfig(dataDir);
21
+ const { remoteCopyMatchesTarget, resolveTargets } = await import("./storage-targets.js");
22
+ const targets = await resolveTargets(dataDir);
23
+ const byId = remote.targetId ? targets.find((candidate) => candidate.id === remote.targetId) : undefined;
24
+ if (byId && await remoteCopyMatchesTarget(dataDir, byId, remote))
25
+ return byId;
26
+ for (const candidate of targets) {
27
+ if (candidate === byId)
28
+ continue;
29
+ if (await remoteCopyMatchesTarget(dataDir, candidate, remote))
30
+ return candidate;
31
+ }
32
+ if (config.remoteStorage) {
33
+ const primary = { id: "primary", ...config.remoteStorage };
34
+ if (await remoteCopyMatchesTarget(dataDir, primary, remote))
35
+ return primary;
36
+ }
37
+ return fail("storage_not_configured", `The exact ${remote.provider} destination holding this copy is no longer configured on this machine. A reused target id or matching bucket name is not enough; restore the original endpoint/account route.`);
38
+ }
39
+ async function ownBucketConnection(dataDir, target, legacyOverrides = {}) {
40
+ const { credentialConfigIdForTarget, endpointOverridesForResolvedTarget } = await import("./storage-targets.js");
41
+ const credentialId = await credentialConfigIdForTarget(dataDir, target);
42
+ if (!credentialId)
43
+ fail("signer_not_configured", `Storage target "${target.id}" is not connected on this machine.`);
44
+ const ownOverrides = await endpointOverridesForResolvedTarget(dataDir, target);
45
+ return {
46
+ credentialId,
47
+ // Callers may still provide the setup wizard's old endpoint override, but
48
+ // it is valid only for the synthesized legacy primary.
49
+ overrides: Object.keys(ownOverrides).length > 0
50
+ ? ownOverrides
51
+ : target.id === "primary" ? legacyOverrides : {},
52
+ };
53
+ }
10
54
  /** Providers this build can fetch back from. GCS uploads but has no download client. */
11
55
  // Every provider with a proven read-back path. "vaultline" reads back through
12
56
  // the account (pullCiphertext, hash-checked); "gcs" through a signed GET on
@@ -15,31 +59,145 @@ import { sha256 } from "./crypto.js";
15
59
  // gdrive joined when its client grew download() — the live Drive drill hit
16
60
  // this stale gate first: uploads landed, the local copy could never leave.
17
61
  const FETCHABLE = new Set(["s3", "r2", "b2", "gcs", "gdrive", "vaultline"]);
18
- export async function offloadable(dataDir) {
62
+ /**
63
+ * The local-cache clock starts only after both facts are true: the archive
64
+ * exists and an off-machine copy has been verified. An old archive uploaded
65
+ * today therefore receives the same complete cache period as a new one.
66
+ */
67
+ export function archiveCacheReferenceTime(record) {
68
+ const created = Date.parse(record.createdAt);
69
+ const verified = isV2(record)
70
+ ? archiveCopies(record)
71
+ .filter((copy) => FETCHABLE.has(copy.provider))
72
+ .map((copy) => Date.parse(copy.verifiedAt))
73
+ .filter(Number.isFinite)
74
+ : [];
75
+ return Math.max(Number.isFinite(created) ? created : Number.NEGATIVE_INFINITY, ...verified);
76
+ }
77
+ function passesAgeScope(record, options) {
78
+ if (options.olderThanDays === undefined)
79
+ return true;
80
+ if (!Number.isFinite(options.olderThanDays) || options.olderThanDays < 0)
81
+ return false;
82
+ const reference = archiveCacheReferenceTime(record);
83
+ return Number.isFinite(reference)
84
+ && reference <= (options.now ?? Date.now()) - options.olderThanDays * 86_400_000;
85
+ }
86
+ function passesOffloadScope(record, options) {
87
+ return (!options.only || options.only.has(record.id)) && passesAgeScope(record, options);
88
+ }
89
+ function physicalBytesOf(info) {
90
+ if (typeof info.nlink === "number" && info.nlink > 1)
91
+ return 0;
92
+ if (typeof info.blocks === "number" && Number.isSafeInteger(info.blocks) && info.blocks >= 0) {
93
+ return info.blocks * 512;
94
+ }
95
+ // Node does not expose st_blocks on every platform. Logical size is the
96
+ // least misleading fallback there; POSIX callers always take the branch
97
+ // above, including sparse files whose physical allocation is genuinely 0.
98
+ return info.size;
99
+ }
100
+ /**
101
+ * Returns one destination shared by every link in a delta chain, but only when
102
+ * every link at that destination is readable by this build. A verified copy
103
+ * on each link is not enough: split A/B copies cannot restore the session.
104
+ */
105
+ function commonReadableDestination(chain) {
106
+ if (chain.length === 1) {
107
+ return archiveCopies(chain[0]).some((copy) => Number.isFinite(Date.parse(copy.verifiedAt)) && FETCHABLE.has(copy.provider))
108
+ ? "self-contained"
109
+ : null;
110
+ }
111
+ let common = null;
112
+ for (const record of chain) {
113
+ const destinations = new Set(archiveCopies(record)
114
+ .filter((copy) => Number.isFinite(Date.parse(copy.verifiedAt)) && FETCHABLE.has(copy.provider))
115
+ .map(verifiedCopyDestinationKey)
116
+ .filter((key) => Boolean(key)));
117
+ if (common === null)
118
+ common = destinations;
119
+ else {
120
+ const previous = common;
121
+ common = new Set([...previous].filter((key) => destinations.has(key)));
122
+ }
123
+ if (common.size === 0)
124
+ return null;
125
+ }
126
+ return [...(common ?? [])][0] ?? null;
127
+ }
128
+ export async function offloadable(dataDir, options = {}) {
19
129
  const archives = await listArchives(dataDir);
20
130
  const candidates = [];
21
131
  const blocked = [];
132
+ // Resolve the graph once so an incomplete chain blocks every local
133
+ // dependency, and an age-limited sweep never removes an old base still
134
+ // needed by a recent retained head.
135
+ const chainsById = new Map();
136
+ const unsafeDependencyIds = new Set();
137
+ const retainedByRecentHead = new Set();
138
+ for (const item of archives) {
139
+ if (!isV2(item))
140
+ continue;
141
+ let chain;
142
+ try {
143
+ const resolved = resolveDeltaChain(archives, item);
144
+ if (!resolved.every(isV2))
145
+ continue;
146
+ chain = resolved;
147
+ }
148
+ catch {
149
+ continue;
150
+ }
151
+ chainsById.set(item.id, chain);
152
+ if (chain.length > 1 && !commonReadableDestination(chain)) {
153
+ for (const link of chain)
154
+ unsafeDependencyIds.add(link.id);
155
+ }
156
+ if (options.olderThanDays !== undefined && !passesAgeScope(item, options) && chain.length > 1) {
157
+ for (const link of chain)
158
+ retainedByRecentHead.add(link.id);
159
+ }
160
+ }
22
161
  for (const record of archives) {
162
+ if (!passesOffloadScope(record, options))
163
+ continue;
23
164
  if (!isV2(record)) {
24
165
  blocked.push({ id: record.id, reason: "archive uses format v1; run `sealkeep migrate` first" });
25
166
  continue;
26
167
  }
27
168
  if (record.offloaded)
28
169
  continue;
29
- const remote = record.remote;
30
- if (!remote?.verifiedAt) {
31
- blocked.push({ id: record.id, reason: "no verified remote copy, so the local one is the only copy" });
32
- continue;
33
- }
34
- if (!FETCHABLE.has(remote.provider)) {
35
- blocked.push({ id: record.id, reason: `${remote.provider} can be uploaded to but not read back by this build, so the local copy has to stay` });
170
+ if (retainedByRecentHead.has(record.id))
36
171
  continue;
37
- }
38
172
  const local = await stat(record.objectPath).catch(() => null);
39
173
  if (!local) {
40
174
  blocked.push({ id: record.id, reason: "no local archive file to remove" });
41
175
  continue;
42
176
  }
177
+ const chain = chainsById.get(record.id);
178
+ if (!chain) {
179
+ blocked.push({ id: record.id, reason: "the archive's delta dependency chain is incomplete, so no safe offload proof exists" });
180
+ continue;
181
+ }
182
+ if (unsafeDependencyIds.has(record.id)) {
183
+ blocked.push({ id: record.id, reason: "verified copies for this delta chain are split or not readable at one destination, so the local copy stays" });
184
+ continue;
185
+ }
186
+ const verified = archiveCopies(record).filter((copy) => Number.isFinite(Date.parse(copy.verifiedAt)));
187
+ if (verified.length === 0) {
188
+ blocked.push({ id: record.id, reason: "no verified remote copy, so the local archive is the only copy" });
189
+ continue;
190
+ }
191
+ const readable = verified.filter((copy) => FETCHABLE.has(copy.provider));
192
+ if (readable.length === 0) {
193
+ blocked.push({ id: record.id, reason: `${verified.map((copy) => copy.provider).join(", ")} cannot be read back by this build, so the local copy has to stay` });
194
+ continue;
195
+ }
196
+ const commonDestination = commonReadableDestination(chain);
197
+ if (!commonDestination) {
198
+ blocked.push({ id: record.id, reason: "verified copies for this delta chain are split or not readable at one destination, so the local copy stays" });
199
+ continue;
200
+ }
43
201
  // The remote is only a copy if the sizes agree. For a provider that stores
44
202
  // the raw ciphertext that means equality; a managed object carries the
45
203
  // self-opening envelope frame around the same ciphertext, so it is larger
@@ -47,17 +205,678 @@ export async function offloadable(dataDir) {
47
205
  // demanding it silently blocked every managed offload. The frame only
48
206
  // adds, so smaller-than-local is still the same corruption signal, and
49
207
  // the true proof either way is the fetch-and-hash below.
50
- const sameSize = remote.provider === "vaultline" ? remote.bytes >= local.size : remote.bytes === local.size;
51
- if (!sameSize) {
52
- blocked.push({ id: record.id, reason: `the remote object is ${remote.bytes} bytes and the local one is ${local.size}; they are not the same archive` });
208
+ const remote = readable.find((copy) => (chain.length === 1 || verifiedCopyDestinationKey(copy) === commonDestination)
209
+ && (copy.provider === "vaultline" ? copy.bytes >= local.size : copy.bytes === local.size));
210
+ if (!remote) {
211
+ blocked.push({ id: record.id, reason: `none of the ${readable.length} readable stored copies has the expected ${local.size} bytes` });
53
212
  continue;
54
213
  }
55
214
  candidates.push({
56
215
  id: record.id, agent: record.source.agent, path: record.source.path,
57
- localBytes: local.size, provider: remote.provider, bucket: remote.bucket, objectKey: remote.objectKey
216
+ localBytes: local.size, allocatedBytes: physicalBytesOf(local),
217
+ provider: remote.provider, bucket: remote.bucket, objectKey: remote.objectKey,
218
+ copyId: remote.copyId
58
219
  });
59
220
  }
60
- return { candidates, blocked, freeableBytes: candidates.reduce((total, item) => total + item.localBytes, 0) };
221
+ return { candidates, blocked, freeableBytes: candidates.reduce((total, item) => total + item.allocatedBytes, 0) };
222
+ }
223
+ const OFFLOAD_STAGES = new Set(["prepared", "linked", "remote-verified", "metadata-committed", "source-unlinked", "purge-authorized"]);
224
+ const OFFLOAD_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
225
+ function offloadJournalDirectory(dataDir) {
226
+ return join(resolve(dataDir), "offload-transactions");
227
+ }
228
+ export function offloadJournalPath(dataDir, id) {
229
+ if (!OFFLOAD_ID.test(id))
230
+ throw new Error("Invalid offload transaction id");
231
+ return join(offloadJournalDirectory(dataDir), `${id}.json`);
232
+ }
233
+ function offloadStagingPath(objectPath, id) {
234
+ return join(dirname(resolve(objectPath)), `.${basename(objectPath)}.sealkeep-offload-${id}.staged`);
235
+ }
236
+ async function syncOffloadDirectory(path) {
237
+ let handle;
238
+ try {
239
+ handle = await open(path, "r");
240
+ await handle.sync();
241
+ }
242
+ catch (error) {
243
+ if (process.platform !== "win32")
244
+ throw error;
245
+ }
246
+ finally {
247
+ await handle?.close();
248
+ }
249
+ }
250
+ async function canonicalNonSymlinkDirectory(path, label) {
251
+ const lexical = resolve(path);
252
+ const info = await lstat(lexical);
253
+ if (!info.isDirectory() || info.isSymbolicLink())
254
+ throw new Error(`${label} is not a non-symlink directory: ${lexical}`);
255
+ const canonical = await realpath(lexical);
256
+ const canonicalInfo = await lstat(canonical);
257
+ if (!canonicalInfo.isDirectory() || canonicalInfo.isSymbolicLink())
258
+ throw new Error(`${label} does not resolve to a trusted directory: ${lexical}`);
259
+ return canonical;
260
+ }
261
+ async function trustedChildDirectory(parent, name, create) {
262
+ const path = join(parent, name);
263
+ let created = false;
264
+ if (create) {
265
+ try {
266
+ await mkdir(path, { mode: 0o700 });
267
+ created = true;
268
+ }
269
+ catch (error) {
270
+ if (error.code !== "EEXIST")
271
+ throw error;
272
+ }
273
+ }
274
+ const info = await lstat(path).catch((error) => {
275
+ if (error.code === "ENOENT" && !create)
276
+ return null;
277
+ throw error;
278
+ });
279
+ if (!info)
280
+ return null;
281
+ if (!info.isDirectory() || info.isSymbolicLink() || await realpath(path) !== path) {
282
+ throw new Error(`Offload transaction directory is not a canonical non-symlink directory: ${path}`);
283
+ }
284
+ if (created) {
285
+ await syncOffloadDirectory(parent);
286
+ await syncOffloadDirectory(path);
287
+ }
288
+ return path;
289
+ }
290
+ async function trustedOffloadJournalDirectory(dataDir, create) {
291
+ const canonicalDataDir = await canonicalNonSymlinkDirectory(dataDir, "Sealkeep data directory");
292
+ return trustedChildDirectory(canonicalDataDir, "offload-transactions", create);
293
+ }
294
+ async function trustedOffloadLockPath(dataDir, key) {
295
+ const journalDirectory = await trustedOffloadJournalDirectory(dataDir, true);
296
+ if (!journalDirectory)
297
+ throw new Error("Could not create the offload transaction directory");
298
+ const lockDirectory = await trustedChildDirectory(journalDirectory, ".locks", true);
299
+ if (!lockDirectory)
300
+ throw new Error("Could not create the offload lock directory");
301
+ const queueDirectory = await trustedChildDirectory(lockDirectory, key, true);
302
+ if (!queueDirectory)
303
+ throw new Error("Could not create the archive offload lock queue");
304
+ return queueDirectory;
305
+ }
306
+ async function assertNoFollowJournal(path) {
307
+ const info = await lstat(path);
308
+ if (!info.isFile() || info.isSymbolicLink())
309
+ throw new Error(`Offload journal is not a non-symlink regular file: ${path}`);
310
+ }
311
+ async function readJournalNoFollow(path) {
312
+ const noFollow = typeof fsConstants.O_NOFOLLOW === "number" ? fsConstants.O_NOFOLLOW : 0;
313
+ const handle = await open(path, fsConstants.O_RDONLY | noFollow);
314
+ try {
315
+ const before = await handle.stat({ bigint: true });
316
+ if (!before.isFile())
317
+ throw new Error(`Offload journal is not a regular file: ${path}`);
318
+ const raw = await handle.readFile("utf8");
319
+ const after = await handle.stat({ bigint: true });
320
+ if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size
321
+ || before.mtimeNs !== after.mtimeNs || before.ctimeNs !== after.ctimeNs) {
322
+ throw new Error(`Offload journal changed while it was being read: ${path}`);
323
+ }
324
+ return raw;
325
+ }
326
+ finally {
327
+ await handle.close();
328
+ }
329
+ }
330
+ async function writeOffloadJournal(transaction, exclusive = false) {
331
+ const directory = await trustedOffloadJournalDirectory(transactionDataDir(transaction), true);
332
+ if (!directory)
333
+ throw new Error("Could not create the offload transaction directory");
334
+ const path = join(directory, `${transaction.id}.json`);
335
+ const temporary = join(directory, `.${basename(path)}.${randomUUID()}.tmp`);
336
+ let handle;
337
+ try {
338
+ handle = await open(temporary, "wx", 0o600);
339
+ await handle.writeFile(`${JSON.stringify(transaction)}\n`, "utf8");
340
+ await handle.sync();
341
+ await handle.close();
342
+ handle = undefined;
343
+ if (exclusive)
344
+ await hardLink(temporary, path);
345
+ else {
346
+ await assertNoFollowJournal(path);
347
+ await rename(temporary, path);
348
+ }
349
+ await syncOffloadDirectory(directory);
350
+ }
351
+ finally {
352
+ await handle?.close().catch(() => undefined);
353
+ await rm(temporary, { force: true }).catch(() => undefined);
354
+ }
355
+ }
356
+ async function removeOffloadJournal(dataDir, id) {
357
+ const directory = await trustedOffloadJournalDirectory(dataDir, false);
358
+ if (!directory)
359
+ throw new Error("Offload transaction directory disappeared before journal cleanup");
360
+ const path = join(directory, `${id}.json`);
361
+ await assertNoFollowJournal(path);
362
+ await unlink(path);
363
+ await syncOffloadDirectory(directory);
364
+ }
365
+ function transactionDataDir(transaction) {
366
+ const value = transaction.dataDir;
367
+ if (!value)
368
+ throw new Error("Offload transaction has no owning data directory");
369
+ return value;
370
+ }
371
+ function liveOffloadTransaction(dataDir, value) {
372
+ Object.defineProperty(value, "dataDir", { value: resolve(dataDir), enumerable: false, configurable: false, writable: false });
373
+ return value;
374
+ }
375
+ async function replaceOffloadJournal(transaction, stage, patch = {}) {
376
+ const next = liveOffloadTransaction(transaction.dataDir, {
377
+ ...transaction,
378
+ ...patch,
379
+ updatedAt: new Date().toISOString(),
380
+ stage,
381
+ });
382
+ await writeOffloadJournal(next);
383
+ return next;
384
+ }
385
+ async function removeOffloadPathDurably(path) {
386
+ await unlink(path);
387
+ await syncOffloadDirectory(dirname(path));
388
+ }
389
+ async function localArchiveIdentity(path) {
390
+ const info = await lstat(path, { bigint: true });
391
+ if (!info.isFile() || info.isSymbolicLink())
392
+ throw new Error(`Local archive is not a regular non-symlink file: ${path}`);
393
+ if (info.size > BigInt(Number.MAX_SAFE_INTEGER) || info.blocks * 512n > BigInt(Number.MAX_SAFE_INTEGER)) {
394
+ throw new Error(`Local archive is too large to measure safely: ${path}`);
395
+ }
396
+ return {
397
+ bytes: Number(info.size),
398
+ allocatedBytes: Number(info.blocks * 512n),
399
+ links: Number(info.nlink),
400
+ dev: info.dev.toString(),
401
+ ino: info.ino.toString(),
402
+ };
403
+ }
404
+ async function localArchiveIdentityOrNull(path) {
405
+ try {
406
+ return await localArchiveIdentity(path);
407
+ }
408
+ catch (error) {
409
+ if (error.code === "ENOENT")
410
+ return null;
411
+ throw error;
412
+ }
413
+ }
414
+ function sameOffloadInode(left, right) {
415
+ return left.dev === right.dev && left.ino === right.ino;
416
+ }
417
+ function sameOffloadFile(left, right) {
418
+ return sameOffloadInode(left, right) && left.bytes === right.bytes;
419
+ }
420
+ function sameOffloadSnapshot(left, right) {
421
+ return sameOffloadFile(left, right)
422
+ && left.allocatedBytes === right.allocatedBytes
423
+ && left.links === right.links;
424
+ }
425
+ function parseOffloadJournal(dataDir, name, raw) {
426
+ const value = JSON.parse(raw);
427
+ const original = value.original;
428
+ if (value.version !== 1
429
+ || typeof value.id !== "string" || !OFFLOAD_ID.test(value.id) || name !== `${value.id}.json`
430
+ || typeof value.archiveId !== "string" || !OFFLOAD_ID.test(value.archiveId)
431
+ || typeof value.objectPath !== "string" || value.objectPath !== resolve(value.objectPath)
432
+ || typeof value.stagingPath !== "string" || value.stagingPath !== offloadStagingPath(value.objectPath, value.id)
433
+ || typeof value.createdAt !== "string" || !Number.isFinite(Date.parse(value.createdAt))
434
+ || typeof value.updatedAt !== "string" || !Number.isFinite(Date.parse(value.updatedAt))
435
+ || !value.stage || !OFFLOAD_STAGES.has(value.stage)
436
+ || !original || !Number.isSafeInteger(original.bytes) || (original.bytes ?? -1) < 0
437
+ || !Number.isSafeInteger(original.allocatedBytes) || (original.allocatedBytes ?? -1) < 0
438
+ || !Number.isSafeInteger(original.links) || (original.links ?? 0) < 1
439
+ || typeof original.dev !== "string" || typeof original.ino !== "string"
440
+ || (value.purgeSnapshot !== undefined && (!Number.isSafeInteger(value.purgeSnapshot.bytes) || value.purgeSnapshot.bytes < 0
441
+ || !Number.isSafeInteger(value.purgeSnapshot.allocatedBytes) || value.purgeSnapshot.allocatedBytes < 0
442
+ || !Number.isSafeInteger(value.purgeSnapshot.links) || value.purgeSnapshot.links < 1
443
+ || typeof value.purgeSnapshot.dev !== "string" || typeof value.purgeSnapshot.ino !== "string"))
444
+ || !value.remote || typeof value.remote.provider !== "string" || typeof value.remote.bucket !== "string" || typeof value.remote.objectKey !== "string"
445
+ || (value.remote.copyId !== undefined && typeof value.remote.copyId !== "string")
446
+ || (value.remote.destinationKey !== undefined && typeof value.remote.destinationKey !== "string")) {
447
+ throw new Error(`Invalid offload transaction journal: ${join(offloadJournalDirectory(dataDir), name)}`);
448
+ }
449
+ return liveOffloadTransaction(dataDir, value);
450
+ }
451
+ async function readOffloadJournals(dataDir) {
452
+ const directory = await trustedOffloadJournalDirectory(dataDir, false);
453
+ if (!directory)
454
+ return [];
455
+ const names = await readdir(directory);
456
+ const journals = [];
457
+ for (const name of names.filter((item) => item.endsWith(".json")).sort()) {
458
+ const path = join(directory, name);
459
+ await assertNoFollowJournal(path);
460
+ journals.push(parseOffloadJournal(dataDir, name, await readJournalNoFollow(path)));
461
+ }
462
+ return journals;
463
+ }
464
+ async function beginOffloadTransaction(dataDir, candidate, record) {
465
+ const original = await localArchiveIdentity(record.objectPath);
466
+ if (original.bytes !== record.cipher.storedBytes || original.bytes !== candidate.localBytes) {
467
+ throw new Error("the local encrypted archive changed while offload was starting; local copy kept");
468
+ }
469
+ const now = new Date().toISOString();
470
+ const id = randomUUID();
471
+ const selected = archiveCopies(record).find((copy) => copy.copyId === candidate.copyId);
472
+ if (!selected)
473
+ throw new Error("the selected verified copy changed before offload could start; local copy kept");
474
+ const destinationKey = verifiedCopyDestinationKey(selected);
475
+ const transaction = liveOffloadTransaction(dataDir, {
476
+ version: 1,
477
+ id,
478
+ archiveId: record.id,
479
+ objectPath: resolve(record.objectPath),
480
+ stagingPath: offloadStagingPath(record.objectPath, id),
481
+ createdAt: now,
482
+ updatedAt: now,
483
+ stage: "prepared",
484
+ original,
485
+ remote: {
486
+ ...(candidate.copyId ? { copyId: candidate.copyId } : {}),
487
+ ...(destinationKey ? { destinationKey } : {}),
488
+ provider: candidate.provider,
489
+ bucket: candidate.bucket,
490
+ objectKey: candidate.objectKey,
491
+ },
492
+ });
493
+ await assertOffloadRecordOwnsPath(dataDir, transaction);
494
+ await writeOffloadJournal(transaction, true);
495
+ return transaction;
496
+ }
497
+ async function linkOffloadStaging(transaction) {
498
+ const before = await localArchiveIdentity(transaction.objectPath);
499
+ if (!sameOffloadFile(before, transaction.original)) {
500
+ throw new Error("the local encrypted archive changed before it could be staged; local copy kept");
501
+ }
502
+ await hardLink(transaction.objectPath, transaction.stagingPath);
503
+ const [source, staged] = await Promise.all([
504
+ localArchiveIdentity(transaction.objectPath),
505
+ localArchiveIdentity(transaction.stagingPath),
506
+ ]);
507
+ if (!sameOffloadFile(source, staged) || !sameOffloadFile(source, transaction.original)) {
508
+ throw new Error("the encrypted archive staging link does not identify the original inode; both paths were kept");
509
+ }
510
+ const { handle: stagedHandle } = await openForFlush(transaction.stagingPath);
511
+ try {
512
+ await stagedHandle.sync();
513
+ }
514
+ finally {
515
+ await stagedHandle.close();
516
+ }
517
+ await syncOffloadDirectory(dirname(transaction.stagingPath));
518
+ return replaceOffloadJournal(transaction, "linked");
519
+ }
520
+ function selectedOffloadCopy(record, transaction) {
521
+ return archiveCopies(record).find((copy) => transaction.remote.copyId
522
+ ? copy.copyId === transaction.remote.copyId
523
+ : copy.provider === transaction.remote.provider
524
+ && copy.bucket === transaction.remote.bucket
525
+ && copy.objectKey === transaction.remote.objectKey) ?? null;
526
+ }
527
+ function offloadMetadataMatches(record, transaction) {
528
+ return record.offloaded?.transactionId === transaction.id
529
+ && record.offloaded.provider === transaction.remote.provider
530
+ && record.offloaded.bucket === transaction.remote.bucket
531
+ && record.offloaded.objectKey === transaction.remote.objectKey;
532
+ }
533
+ function offloadMetadataFinalized(record, transaction) {
534
+ const offloaded = record.offloaded;
535
+ return Boolean(offloaded
536
+ && offloaded.transactionId === undefined
537
+ && offloaded.provider === transaction.remote.provider
538
+ && offloaded.bucket === transaction.remote.bucket
539
+ && offloaded.objectKey === transaction.remote.objectKey
540
+ && offloaded.localBytes === transaction.original.bytes
541
+ && Number.isSafeInteger(offloaded.freedBytes)
542
+ && (offloaded.freedBytes ?? -1) >= 0);
543
+ }
544
+ async function verifyOffloadRemoteChain(dataDir, record, transaction, options) {
545
+ const selected = selectedOffloadCopy(record, transaction);
546
+ if (!selected)
547
+ throw new Error("the verified restore copy changed while offload was being checked; local copy kept");
548
+ if (!Number.isFinite(Date.parse(selected.verifiedAt)) || !FETCHABLE.has(selected.provider)) {
549
+ throw new Error("the selected restore copy is no longer verified and readable; local copy kept");
550
+ }
551
+ const archives = await listArchives(dataDir);
552
+ const liveRecord = archives.find((item) => item.id === record.id && isV2(item));
553
+ if (!liveRecord)
554
+ throw new Error("the archive disappeared while its remote dependency chain was being checked; local copy kept");
555
+ const resolved = resolveDeltaChain(archives, liveRecord);
556
+ if (!resolved.every(isV2))
557
+ throw new Error("the archive's remote dependency chain is incomplete; local copy kept");
558
+ const chain = resolved;
559
+ const selectedDestination = transaction.remote.destinationKey ?? verifiedCopyDestinationKey(selected);
560
+ if (chain.length > 1 && !selectedDestination) {
561
+ throw new Error("the archive's remote dependency chain has no stable common destination; local copy kept");
562
+ }
563
+ for (const link of chain) {
564
+ const copy = link.id === liveRecord.id
565
+ ? selectedOffloadCopy(link, transaction)
566
+ : archiveCopies(link).find((candidate) => Number.isFinite(Date.parse(candidate.verifiedAt))
567
+ && FETCHABLE.has(candidate.provider)
568
+ && verifiedCopyDestinationKey(candidate) === selectedDestination) ?? null;
569
+ if (!copy || !Number.isFinite(Date.parse(copy.verifiedAt)) || !FETCHABLE.has(copy.provider)
570
+ || (chain.length > 1 && verifiedCopyDestinationKey(copy) !== selectedDestination)) {
571
+ throw new Error(`the complete remote delta chain is no longer readable at one destination (${link.id}); local copy kept`);
572
+ }
573
+ let stream = await fetchCiphertextStream(dataDir, link, options.client, copy, options.overrides, options.signal);
574
+ if (options.maxBytesPerSecond !== undefined) {
575
+ stream = paceBackgroundByteStream(dataDir, stream, options.maxBytesPerSecond, { signal: options.signal });
576
+ }
577
+ const fetched = await digestChunks(stream);
578
+ if (fetched.sha256Hex !== link.cipher.ciphertextSha256 || fetched.bytes !== link.cipher.storedBytes) {
579
+ throw new Error(`what the bucket returned is not this archive (${link.id}) in the required delta chain; local copy kept`);
580
+ }
581
+ }
582
+ }
583
+ async function markOffloadMetadata(dataDir, transaction) {
584
+ const config = await readConfig(dataDir);
585
+ await mutateArchiveRecord(config.storage.root, transaction.archiveId, (current) => {
586
+ if (!isV2(current) || resolve(current.objectPath) !== transaction.objectPath || !selectedOffloadCopy(current, transaction)) {
587
+ throw new Error("archive metadata changed while offload was being committed; local copy kept");
588
+ }
589
+ if (current.offloaded && !offloadMetadataMatches(current, transaction)) {
590
+ throw new Error("another offload already changed this archive record; local copy kept");
591
+ }
592
+ return {
593
+ ...current,
594
+ offloaded: {
595
+ at: current.offloaded?.at ?? new Date().toISOString(),
596
+ provider: transaction.remote.provider,
597
+ bucket: transaction.remote.bucket,
598
+ objectKey: transaction.remote.objectKey,
599
+ transactionId: transaction.id,
600
+ localBytes: transaction.original.bytes,
601
+ freedBytes: 0,
602
+ },
603
+ };
604
+ });
605
+ return replaceOffloadJournal(transaction, "metadata-committed");
606
+ }
607
+ async function unlinkOffloadSource(transaction) {
608
+ const [source, staged] = await Promise.all([
609
+ localArchiveIdentity(transaction.objectPath),
610
+ localArchiveIdentity(transaction.stagingPath),
611
+ ]);
612
+ if (!sameOffloadFile(source, staged) || !sameOffloadFile(source, transaction.original)) {
613
+ throw new Error("the local archive path changed before offload could remove it; every file was kept");
614
+ }
615
+ await removeOffloadPathDurably(transaction.objectPath);
616
+ return replaceOffloadJournal(transaction, "source-unlinked");
617
+ }
618
+ async function finalizeOffloadMetadata(dataDir, transaction, freedBytes) {
619
+ const config = await readConfig(dataDir);
620
+ await mutateArchiveRecord(config.storage.root, transaction.archiveId, (current) => {
621
+ if (!isV2(current) || !offloadMetadataMatches(current, transaction)) {
622
+ throw new Error("offload completed but its archive record changed; the recovery journal was retained");
623
+ }
624
+ const { transactionId: _transactionId, ...offloaded } = current.offloaded;
625
+ return { ...current, offloaded: { ...offloaded, freedBytes, localBytes: transaction.original.bytes } };
626
+ });
627
+ await removeOffloadJournal(dataDir, transaction.id);
628
+ return freedBytes;
629
+ }
630
+ async function clearOwnedOffloadMetadata(dataDir, transaction) {
631
+ const config = await readConfig(dataDir);
632
+ await mutateArchiveRecord(config.storage.root, transaction.archiveId, (current) => {
633
+ if (!isV2(current) || !offloadMetadataMatches(current, transaction))
634
+ return current;
635
+ const { offloaded: _offloaded, ...rest } = current;
636
+ return rest;
637
+ });
638
+ }
639
+ async function assertOffloadRecordOwnsPath(dataDir, transaction) {
640
+ const config = await readConfig(dataDir);
641
+ const record = (await listArchives(dataDir)).find((item) => item.id === transaction.archiveId && isV2(item));
642
+ const lexicalRoot = resolve(config.storage.root);
643
+ const expectedObject = join(lexicalRoot, `${transaction.archiveId}.skarchive`);
644
+ if (!record || resolve(record.objectPath) !== transaction.objectPath
645
+ || transaction.objectPath !== expectedObject
646
+ || dirname(transaction.stagingPath) !== lexicalRoot
647
+ || transaction.stagingPath !== offloadStagingPath(transaction.objectPath, transaction.id)) {
648
+ throw new Error("archive record no longer owns the path named by its offload journal; every path and the journal were kept");
649
+ }
650
+ // Resolve only the trusted configured root, then validate each descendant
651
+ // one component at a time. This permits platform aliases such as macOS
652
+ // /var -> /private/var without trusting a symlink inserted inside archives/.
653
+ const canonicalRoot = await realpath(lexicalRoot);
654
+ const canonicalObject = join(canonicalRoot, basename(transaction.objectPath));
655
+ const canonicalStaging = join(canonicalRoot, basename(transaction.stagingPath));
656
+ await validateNoFollowPath(canonicalObject, [canonicalRoot], { leaf: "missing-or-file" });
657
+ await validateNoFollowPath(canonicalStaging, [canonicalRoot], { leaf: "missing-or-file" });
658
+ return record;
659
+ }
660
+ async function rollbackOffloadTransaction(dataDir, transaction) {
661
+ await assertOffloadRecordOwnsPath(dataDir, transaction);
662
+ const source = await localArchiveIdentityOrNull(transaction.objectPath);
663
+ const staged = await localArchiveIdentityOrNull(transaction.stagingPath);
664
+ if (source && staged) {
665
+ if (!sameOffloadFile(source, transaction.original)
666
+ || !sameOffloadFile(staged, transaction.original)
667
+ || !sameOffloadFile(source, staged)) {
668
+ throw new Error(`Offload conflict kept both ${transaction.objectPath} and ${transaction.stagingPath}`);
669
+ }
670
+ await removeOffloadPathDurably(transaction.stagingPath);
671
+ }
672
+ else if (!source && staged) {
673
+ if (!sameOffloadFile(staged, transaction.original)) {
674
+ throw new Error(`Offload staging conflict kept ${transaction.stagingPath}`);
675
+ }
676
+ await hardLink(transaction.stagingPath, transaction.objectPath);
677
+ await syncOffloadDirectory(dirname(transaction.objectPath));
678
+ const restored = await localArchiveIdentity(transaction.objectPath);
679
+ if (!sameOffloadFile(restored, transaction.original)) {
680
+ throw new Error(`Offload rollback could not prove the restored inode at ${transaction.objectPath}; both names were kept`);
681
+ }
682
+ await removeOffloadPathDurably(transaction.stagingPath);
683
+ }
684
+ else if (source && !staged) {
685
+ if (!sameOffloadFile(source, transaction.original)) {
686
+ throw new Error(`Offload source conflict kept ${transaction.objectPath} and its transaction journal`);
687
+ }
688
+ }
689
+ else if (!source && !staged) {
690
+ throw new Error("Offload transaction lost both local names before authorization");
691
+ }
692
+ const surviving = await localArchiveIdentityOrNull(transaction.objectPath);
693
+ if (!surviving || !sameOffloadFile(surviving, transaction.original)) {
694
+ throw new Error(`Offload rollback did not retain the exact source inode at ${transaction.objectPath}; metadata and journal were kept`);
695
+ }
696
+ await clearOwnedOffloadMetadata(dataDir, transaction);
697
+ await removeOffloadJournal(dataDir, transaction.id);
698
+ }
699
+ /**
700
+ * Remove the final Sealkeep name while holding the exact inode open. The
701
+ * post-unlink fstat is the truthful answer to "were blocks actually freed?":
702
+ * any pre-existing or concurrently added hard link makes that answer zero.
703
+ */
704
+ async function purgeOffloadStaging(transaction, beforeUnlink) {
705
+ const expected = transaction.purgeSnapshot;
706
+ if (!expected)
707
+ throw new Error("offload purge was not authorised with a final inode snapshot");
708
+ const handle = await open(transaction.stagingPath, "r");
709
+ try {
710
+ const beforeInfo = await handle.stat({ bigint: true });
711
+ if (!beforeInfo.isFile() || beforeInfo.size > BigInt(Number.MAX_SAFE_INTEGER)
712
+ || beforeInfo.blocks * 512n > BigInt(Number.MAX_SAFE_INTEGER)) {
713
+ throw new Error("offload staging is not a measurable regular file at the purge boundary");
714
+ }
715
+ const before = {
716
+ bytes: Number(beforeInfo.size), allocatedBytes: Number(beforeInfo.blocks * 512n), links: Number(beforeInfo.nlink),
717
+ dev: beforeInfo.dev.toString(), ino: beforeInfo.ino.toString(),
718
+ };
719
+ const named = await localArchiveIdentity(transaction.stagingPath);
720
+ if (!sameOffloadSnapshot(before, expected) || !sameOffloadSnapshot(named, expected)) {
721
+ throw new Error("offload staging changed after purge authorization; it was retained");
722
+ }
723
+ await beforeUnlink?.(transaction);
724
+ await unlink(transaction.stagingPath);
725
+ await syncOffloadDirectory(dirname(transaction.stagingPath));
726
+ const afterInfo = await handle.stat({ bigint: true });
727
+ if (afterInfo.dev.toString() !== expected.dev || afterInfo.ino.toString() !== expected.ino
728
+ || afterInfo.size !== BigInt(expected.bytes)) {
729
+ throw new Error("offload lost identity of the staged inode while removing its name");
730
+ }
731
+ const remainingLinks = Number(afterInfo.nlink);
732
+ return remainingLinks === 0 ? Number(afterInfo.blocks * 512n) : 0;
733
+ }
734
+ finally {
735
+ await handle.close();
736
+ }
737
+ }
738
+ async function completeOffloadTransaction(dataDir, transaction, options) {
739
+ let current = transaction;
740
+ let record = await assertOffloadRecordOwnsPath(dataDir, current);
741
+ let source = await localArchiveIdentityOrNull(current.objectPath);
742
+ let staged = await localArchiveIdentityOrNull(current.stagingPath);
743
+ if (!staged && source && current.stage === "prepared") {
744
+ current = await linkOffloadStaging(current);
745
+ await options.operations?.afterLinked?.(current);
746
+ source = await localArchiveIdentityOrNull(current.objectPath);
747
+ staged = await localArchiveIdentityOrNull(current.stagingPath);
748
+ }
749
+ if (!staged || (source && !sameOffloadFile(source, staged)) || !sameOffloadFile(staged, current.original)) {
750
+ throw new Error("offload staging no longer identifies the exact local archive; every surviving path was kept");
751
+ }
752
+ const refreshedRecord = (await listArchives(dataDir)).find((item) => item.id === current.archiveId && isV2(item));
753
+ if (!refreshedRecord)
754
+ throw new Error("archive record disappeared while offload was proving its remote copy");
755
+ record = refreshedRecord;
756
+ if (!offloadMetadataMatches(record, current))
757
+ current = await markOffloadMetadata(dataDir, current);
758
+ else if (current.stage !== "metadata-committed" && current.stage !== "source-unlinked" && current.stage !== "purge-authorized") {
759
+ current = await replaceOffloadJournal(current, "metadata-committed");
760
+ }
761
+ await options.operations?.afterMetadata?.(current);
762
+ source = await localArchiveIdentityOrNull(current.objectPath);
763
+ staged = await localArchiveIdentityOrNull(current.stagingPath);
764
+ if (source) {
765
+ if (!staged || !sameOffloadFile(source, staged) || !sameOffloadFile(source, current.original))
766
+ throw new Error("local archive path was replaced during offload; both files were kept");
767
+ current = await unlinkOffloadSource(current);
768
+ }
769
+ else if (current.stage !== "source-unlinked" && current.stage !== "purge-authorized") {
770
+ current = await replaceOffloadJournal(current, "source-unlinked");
771
+ }
772
+ await options.operations?.afterSourceUnlinked?.(current);
773
+ // The irreversible boundary is removal of the staging name below. Re-read
774
+ // and hash EVERY remote dependency at the frozen common destination here,
775
+ // after all potentially slow local metadata work. In an uninterrupted
776
+ // attempt this is exactly one full streamed read per chain link.
777
+ await verifyOffloadRemoteChain(dataDir, record, current, options);
778
+ current = await replaceOffloadJournal(current, "remote-verified");
779
+ await options.operations?.afterRemoteVerified?.(current);
780
+ const finalStaged = await localArchiveIdentity(current.stagingPath);
781
+ if (!sameOffloadFile(finalStaged, current.original)) {
782
+ throw new Error("offload staging changed at the final purge boundary; it was retained");
783
+ }
784
+ await recordAudit(dataDir, "archive.offload", "allowed", {
785
+ archiveId: current.archiveId,
786
+ transactionId: current.id,
787
+ bytes: current.original.bytes,
788
+ authorizedPhysicalBytes: finalStaged.links === 1 ? finalStaged.allocatedBytes : 0,
789
+ finalLinkCount: finalStaged.links,
790
+ bucket: current.remote.bucket,
791
+ });
792
+ current = await replaceOffloadJournal(current, "purge-authorized", { purgeSnapshot: finalStaged });
793
+ await options.operations?.afterPurgeAuthorized?.(current);
794
+ const freedBytes = await purgeOffloadStaging(current, options.operations?.beforeStagingUnlink);
795
+ await options.operations?.afterStagingPurged?.(current);
796
+ await finalizeOffloadMetadata(dataDir, current, freedBytes);
797
+ const candidate = {
798
+ id: current.archiveId,
799
+ agent: record.source.agent,
800
+ path: record.source.path,
801
+ localBytes: current.original.bytes,
802
+ allocatedBytes: freedBytes,
803
+ provider: current.remote.provider,
804
+ bucket: current.remote.bucket,
805
+ objectKey: current.remote.objectKey,
806
+ ...(current.remote.copyId ? { copyId: current.remote.copyId } : {}),
807
+ };
808
+ return { candidate, freedBytes };
809
+ }
810
+ export async function reconcileOffloadTransactions(dataDir, options = {}) {
811
+ const completed = [];
812
+ const failed = [];
813
+ let freedBytes = 0;
814
+ for (let transaction of await readOffloadJournals(dataDir)) {
815
+ const lockKey = createHash("sha256").update(transaction.archiveId).digest("hex");
816
+ const lock = await acquireDurableTicketLock(await trustedOffloadLockPath(dataDir, lockKey), `archive offload ${transaction.archiveId}`, { timeoutMs: 30_000, signal: options.signal });
817
+ try {
818
+ const record = (await listArchives(dataDir)).find((item) => item.id === transaction.archiveId && isV2(item));
819
+ await assertOffloadRecordOwnsPath(dataDir, transaction);
820
+ const source = await localArchiveIdentityOrNull(transaction.objectPath);
821
+ const staged = await localArchiveIdentityOrNull(transaction.stagingPath);
822
+ const ownsMetadata = Boolean(record && offloadMetadataMatches(record, transaction));
823
+ // The record update can reach disk before journal cleanup. That is a
824
+ // completed transaction, not a rollback candidate; both local names
825
+ // must already be gone before this stale journal is retired.
826
+ if (record && offloadMetadataFinalized(record, transaction)) {
827
+ if (source || staged)
828
+ throw new Error("finalized offload metadata conflicts with a surviving local transaction path");
829
+ const released = record.offloaded?.freedBytes ?? 0;
830
+ await removeOffloadJournal(dataDir, transaction.id);
831
+ freedBytes += released;
832
+ completed.push({
833
+ id: transaction.archiveId, agent: record.source.agent, path: record.source.path,
834
+ localBytes: transaction.original.bytes, allocatedBytes: released,
835
+ provider: transaction.remote.provider, bucket: transaction.remote.bucket, objectKey: transaction.remote.objectKey,
836
+ ...(transaction.remote.copyId ? { copyId: transaction.remote.copyId } : {}),
837
+ });
838
+ continue;
839
+ }
840
+ if (!ownsMetadata) {
841
+ await rollbackOffloadTransaction(dataDir, transaction);
842
+ continue;
843
+ }
844
+ if (!source && !staged) {
845
+ if (transaction.stage !== "purge-authorized") {
846
+ throw new Error("both local names disappeared before a durable purge authorization; transaction retained for repair");
847
+ }
848
+ // The process died before its post-unlink fstat result reached durable
849
+ // metadata. A hard link may have appeared in the syscall-sized gap,
850
+ // so the only honest recovery metric is the conservative lower bound.
851
+ const released = 0;
852
+ await finalizeOffloadMetadata(dataDir, transaction, released);
853
+ freedBytes += released;
854
+ completed.push({
855
+ id: transaction.archiveId, agent: record.source.agent, path: record.source.path,
856
+ localBytes: transaction.original.bytes, allocatedBytes: released,
857
+ provider: transaction.remote.provider, bucket: transaction.remote.bucket, objectKey: transaction.remote.objectKey,
858
+ ...(transaction.remote.copyId ? { copyId: transaction.remote.copyId } : {}),
859
+ });
860
+ continue;
861
+ }
862
+ const outcome = await completeOffloadTransaction(dataDir, transaction, options);
863
+ completed.push(outcome.candidate);
864
+ freedBytes += outcome.freedBytes;
865
+ }
866
+ catch (error) {
867
+ if (options.signal?.aborted)
868
+ throw options.signal.reason ?? error;
869
+ try {
870
+ await rollbackOffloadTransaction(dataDir, transaction);
871
+ }
872
+ catch { /* Every surviving name and the journal remain for a later repair. */ }
873
+ failed.push({ id: transaction.archiveId, error: error instanceof Error ? error.message : "could not reconcile offload" });
874
+ }
875
+ finally {
876
+ await lock.release().catch(() => undefined);
877
+ }
878
+ }
879
+ return { completed, failed, freedBytes };
61
880
  }
62
881
  /**
63
882
  * Removes local archives that a bucket has a verified copy of.
@@ -67,51 +886,50 @@ export async function offloadable(dataDir) {
67
886
  * the deletion of something that has since become the only copy.
68
887
  */
69
888
  export async function offloadArchives(dataDir, options = {}) {
70
- const all = await offloadable(dataDir);
71
- // `only` narrows the sweep — the upload run offloads each archive as its
72
- // verify lands rather than re-walking the whole vault per item.
73
- const candidates = options.only ? all.candidates.filter((candidate) => options.only.has(candidate.id)) : all.candidates;
889
+ const recovered = options.confirm
890
+ ? await reconcileOffloadTransactions(dataDir, options)
891
+ : { completed: [], failed: [], freedBytes: 0 };
892
+ const all = await offloadable(dataDir, options);
893
+ const recoveryFailures = new Set(recovered.failed.map((item) => item.id));
894
+ const candidates = all.candidates.filter((candidate) => !recoveryFailures.has(candidate.id));
74
895
  const blocked = all.blocked;
75
- const freeableBytes = candidates.reduce((total, item) => total + item.localBytes, 0);
896
+ const freeableBytes = candidates.reduce((total, item) => total + item.allocatedBytes, 0);
76
897
  if (!options.confirm)
77
898
  return { mode: "preview", wouldOffload: candidates, blocked, freedBytes: freeableBytes };
78
- const archives = new Map((await listArchives(dataDir)).map((record) => [record.id, record]));
79
- const offloaded = [];
80
- const failed = [];
81
- const config = await readConfig(dataDir);
899
+ const offloaded = [...recovered.completed];
900
+ const failed = [...recovered.failed];
901
+ let freedBytes = recovered.freedBytes;
82
902
  for (const candidate of candidates) {
83
- const record = archives.get(candidate.id);
84
- if (!record || !isV2(record))
85
- continue;
903
+ options.signal?.throwIfAborted();
904
+ const lockKey = createHash("sha256").update(candidate.id).digest("hex");
905
+ const lock = await acquireDurableTicketLock(await trustedOffloadLockPath(dataDir, lockKey), `archive offload ${candidate.id}`, { timeoutMs: 30_000, signal: options.signal });
906
+ let transaction = null;
86
907
  try {
87
- // Prove the bytes are really there, now, before removing the local ones.
88
- // The verifiedAt stamp says they were there once; this asks again, because
89
- // between then and now is exactly where a bucket lifecycle rule lives.
90
- // The one honest exception: an archive whose verify landed seconds ago in
91
- // the same run — between then and now is milliseconds, and re-downloading
92
- // a gigabyte to re-prove it would double the egress of every push.
93
- if (!options.freshlyVerified?.has(candidate.id)) {
94
- const fetched = await fetchCiphertext(dataDir, record, options.client);
95
- if (sha256(fetched) !== record.cipher.ciphertextSha256) {
96
- failed.push({ id: candidate.id, error: "what the bucket returned is not this archive; local copy kept" });
97
- continue;
98
- }
99
- }
100
- await rm(record.objectPath, { force: true });
101
- const next = {
102
- ...record,
103
- offloaded: { at: new Date().toISOString(), provider: candidate.provider, bucket: candidate.bucket, objectKey: candidate.objectKey }
104
- };
105
- await writeRecord(config.storage.root, next);
106
- await recordAudit(dataDir, "archive.offload", "allowed", { archiveId: candidate.id, bytes: candidate.localBytes, bucket: candidate.bucket });
107
- offloaded.push(candidate);
908
+ const record = (await listArchives(dataDir)).find((item) => item.id === candidate.id && isV2(item));
909
+ if (!record || record.offloaded)
910
+ continue;
911
+ transaction = await beginOffloadTransaction(dataDir, candidate, record);
912
+ await options.operations?.afterJournal?.(transaction);
913
+ const outcome = await completeOffloadTransaction(dataDir, transaction, options);
914
+ offloaded.push(outcome.candidate);
915
+ freedBytes += outcome.freedBytes;
108
916
  }
109
917
  catch (error) {
110
- await recordAudit(dataDir, "archive.offload", "denied", { archiveId: candidate.id });
918
+ if (options.signal?.aborted)
919
+ throw options.signal.reason ?? error;
920
+ if (transaction)
921
+ await rollbackOffloadTransaction(dataDir, transaction).catch(() => undefined);
922
+ await recordAudit(dataDir, "archive.offload", "denied", {
923
+ archiveId: candidate.id,
924
+ ...(transaction ? { transactionId: transaction.id } : {}),
925
+ });
111
926
  failed.push({ id: candidate.id, error: error instanceof Error ? error.message : "could not offload" });
112
927
  }
928
+ finally {
929
+ await lock.release().catch(() => undefined);
930
+ }
113
931
  }
114
- return { mode: "apply", offloaded, failed, blocked, freedBytes: offloaded.reduce((total, item) => total + item.localBytes, 0) };
932
+ return { mode: "apply", offloaded, failed, blocked, freedBytes };
115
933
  }
116
934
  /**
117
935
  * Downloads an archive's ciphertext from the bucket it was offloaded to.
@@ -122,103 +940,115 @@ export async function offloadArchives(dataDir, options = {}) {
122
940
  * not be recovered by the person who owns it — which would turn a space-saving
123
941
  * feature into data loss. The flag is set for this call only.
124
942
  */
125
- async function fetchCiphertext(dataDir, record, client) {
126
- const config = await readConfig(dataDir);
127
- const remote = record.remote ?? record.offloaded;
943
+ export async function fetchCiphertextStream(dataDir, record, client, selectedCopy, overrides = {}, signal, idleMs, maxBytesPerSecond) {
944
+ return byteStreamWithIdleTimeout(async (readSignal) => {
945
+ const remote = selectedCopy ?? archiveCopies(record)[0] ?? record.offloaded;
946
+ const managedFrame = remote?.provider === "vaultline" && !client && remote.layout?.kind !== "chunks";
947
+ const stream = await openCiphertextStream(dataDir, record, client, selectedCopy, overrides, readSignal, maxBytesPerSecond);
948
+ // Managed frames are paced before their metadata is stripped. Other
949
+ // providers expose raw ciphertext, which is paced exactly once here.
950
+ return maxBytesPerSecond === undefined || managedFrame ? stream
951
+ : paceBackgroundByteStream(dataDir, stream, maxBytesPerSecond, { signal: readSignal });
952
+ }, { signal, idleMs });
953
+ }
954
+ async function openCiphertextStream(dataDir, record, client, selectedCopy, overrides, signal, maxBytesPerSecond) {
955
+ const remote = selectedCopy ?? archiveCopies(record)[0] ?? record.offloaded;
128
956
  if (!remote)
129
957
  fail("archive_not_found", `Archive ${record.id} has no remote copy to fetch`, { archiveId: record.id });
130
958
  // A managed copy reads back through the account, not a provider credential:
131
959
  // pullCiphertext hash-checks what it downloads before returning it. Branch
132
960
  // on the record's own provider — a vault can hold managed history alongside
133
961
  // a later own-bucket configuration, and each copy fetches its own way.
134
- if (remote.provider === "vaultline" && !client && record.remote?.layout?.kind !== "chunks") {
135
- const { pullCiphertext, unframeObject } = await import("./cloud.js");
136
- const { ciphertext: blob } = await pullCiphertext(dataDir, record.id);
962
+ if (remote.provider === "vaultline" && !client && remote.layout?.kind !== "chunks") {
963
+ const { pullCiphertextStream } = await import("./cloud.js");
137
964
  // A managed object is envelope-framed — the self-carrying wrapper push
138
965
  // added so a bare object can be opened anywhere. What the local record
139
966
  // hashes is the ciphertext INSIDE the frame, so that is what this must
140
967
  // return; handing back the framed bytes made every integrity check
141
968
  // compare an apple to the crate it shipped in. A plain-mode object has
142
969
  // no frame and IS the stored bytes.
143
- const { envelope, ciphertext } = unframeObject(blob);
144
- return envelope ? ciphertext : blob;
970
+ return pullCiphertextStream(dataDir, record.id, undefined, { unframe: true, signal, maxBytesPerSecond });
145
971
  }
146
972
  // Chunk-folder remotes fetch object by object, each hash-checked against
147
973
  // the envelope's own chunk headers — the layout was made for exactly this.
148
974
  // A managed folder reads back through the account's download leases; an own
149
975
  // bucket reads through the provider's object verbs.
150
- if (record.remote?.layout?.kind === "chunks") {
151
- const { fetchChunkFolder } = await import("./chunk-store.js");
976
+ if (remote.layout?.kind === "chunks") {
977
+ const { fetchChunkFolderStream } = await import("./chunk-store.js");
152
978
  let folderClient;
153
979
  if (client)
154
980
  folderClient = client;
155
- else if (record.remote.provider === "vaultline") {
981
+ else if (remote.provider === "vaultline") {
156
982
  folderClient = (await import("./managed-chunks.js")).managedChunkClient(dataDir);
157
983
  }
158
984
  else {
159
- const target = config.remoteStorage ?? fail("storage_not_configured", "This vault has no storage configured, so an offloaded archive cannot be fetched back");
160
- folderClient = await uploadClientFromStore(dataDir, config.vaultId, target, {}, undefined);
985
+ const named = await configuredTargetForRemote(dataDir, remote);
986
+ if (named.provider === "vaultline" || !named.bucket)
987
+ fail("storage_not_configured", "This chunk copy has no usable bucket target on this machine");
988
+ const target = { provider: named.provider, bucket: named.bucket, prefix: named.prefix ?? "sealkeep", ...(named.region ? { region: named.region } : {}) };
989
+ const connection = await ownBucketConnection(dataDir, named, overrides);
990
+ folderClient = await uploadClientFromStore(dataDir, connection.credentialId, target, connection.overrides, undefined);
161
991
  }
162
992
  if (typeof folderClient.getObject !== "function")
163
- fail("provider_unsupported", `${record.remote.provider} cannot fetch chunk folders with this build`);
164
- return fetchChunkFolder(folderClient, record.remote, record.envelope);
993
+ fail("provider_unsupported", `${remote.provider} cannot fetch chunk folders with this build`);
994
+ return fetchChunkFolderStream(folderClient, remote, record.envelope, { signal });
165
995
  }
166
996
  // The record routes its own read: provider and bucket come from the copy
167
997
  // itself, the prefix from its object key, and a configured target (matched
168
998
  // by targetId, then by provider+bucket) fills in the rest. config.remoteStorage
169
999
  // is only the last resort — a multi-target vault has none.
170
- const target = await (async () => {
171
- if (config.remoteStorage && config.remoteStorage.provider === remote.provider && config.remoteStorage.bucket === remote.bucket)
172
- return config.remoteStorage;
173
- try {
174
- const { resolveTargets } = await import("./storage-targets.js");
175
- const targets = await resolveTargets(dataDir);
176
- const stampedId = remote.targetId;
177
- const match = (stampedId ? targets.find((candidate) => candidate.id === stampedId) : undefined)
178
- ?? targets.find((candidate) => candidate.provider === remote.provider && candidate.bucket === remote.bucket);
179
- if (match) {
180
- const derivedPrefix = remote.objectKey.split("/").slice(0, -1).join("/");
181
- return { provider: match.provider, bucket: match.bucket ?? remote.bucket, prefix: match.prefix ?? (derivedPrefix || "vaultline"), region: match.region };
182
- }
183
- }
184
- catch { /* fall through */ }
185
- if (config.remoteStorage)
186
- return config.remoteStorage;
187
- return fail("storage_not_configured", "This vault has no storage configured, so an offloaded archive cannot be fetched back");
188
- })();
189
- if (target.provider === "vaultline")
1000
+ const named = await configuredTargetForRemote(dataDir, remote);
1001
+ if (named.provider === "vaultline" || !named.bucket)
190
1002
  fail("storage_not_configured", "This managed copy should have taken the managed read path");
191
- const resolved = client ?? await uploadClientFromStore(dataDir, config.vaultId, target, {}, undefined);
192
- const lease = createActiveLease(target, {
193
- archiveId: record.id, ciphertextSha256: record.cipher.ciphertextSha256, bytes: record.cipher.storedBytes
194
- }, { ...process.env, SEALKEEP_ENABLE_SIGNER: "1" });
1003
+ const derivedPrefix = remote.objectKey.split("/").slice(0, -1).join("/");
1004
+ const target = {
1005
+ provider: named.provider,
1006
+ bucket: named.bucket,
1007
+ prefix: named.prefix ?? (derivedPrefix || "sealkeep"),
1008
+ ...(named.region ? { region: named.region } : {}),
1009
+ };
1010
+ const connection = client ? null : await ownBucketConnection(dataDir, named, overrides);
1011
+ const resolved = client ?? await uploadClientFromStore(dataDir, connection.credentialId, target, connection.overrides, undefined);
1012
+ const lease = { ...createActiveLease(target, {
1013
+ archiveId: record.id, ciphertextSha256: record.cipher.ciphertextSha256, bytes: record.cipher.storedBytes
1014
+ }, { ...process.env, SEALKEEP_ENABLE_SIGNER: "1" }), objectKey: remote.objectKey };
195
1015
  const readable = resolved;
1016
+ if (typeof readable.downloadStream === "function")
1017
+ return readable.downloadStream(lease, { signal });
196
1018
  if (typeof readable.download !== "function") {
197
1019
  fail("provider_unsupported", `${target.provider} cannot be read back by this build, so this archive cannot be fetched`);
198
1020
  }
199
- return readable.download(lease);
1021
+ // Small injected/test clients predate the stream contract. Never invoke a
1022
+ // whole-Buffer client for a large archive: fail before it can allocate.
1023
+ if (record.cipher.storedBytes > LEGACY_BUFFER_LIMIT_BYTES) {
1024
+ fail("provider_unsupported", `${target.provider} cannot stream this ${record.cipher.storedBytes}-byte archive with the installed client`, { archiveId: record.id });
1025
+ }
1026
+ const buffered = await readable.download(lease, { signal });
1027
+ if (buffered.length > LEGACY_BUFFER_LIMIT_BYTES) {
1028
+ fail("provider_unsupported", `${target.provider} returned more than the bounded compatibility limit`, { archiveId: record.id });
1029
+ }
1030
+ return oneChunk(buffered);
200
1031
  }
201
1032
  /**
202
1033
  * One managed record checked against the account's rows. A single-object copy
203
1034
  * is one row at the recorded size; a chunk-folder copy is its whole family —
204
1035
  * every chunk row present, the envelope sidecar present, and the chunk bytes
205
1036
  * summing to the verified total. The proof standard does not soften because
206
- * the object count went up: any missing member is "do not trash the original".
1037
+ * the object count went up: any missing member is "do not reclaim the original".
207
1038
  */
208
- async function managedCopyCheck(dataDir, record, rows, checkedAt) {
209
- const remote = record.remote;
1039
+ async function managedCopyCheck(dataDir, record, remote, rows, checkedAt) {
210
1040
  if (remote.layout?.kind === "chunks") {
211
1041
  const { managedChunkRefs } = await import("./managed-chunks.js");
212
1042
  const refs = managedChunkRefs(record.id, remote.layout.chunkCount);
213
1043
  const missing = refs.chunks.filter((ref) => !rows.has(ref)).length + (rows.has(refs.envelope) ? 0 : 1);
214
1044
  if (missing > 0) {
215
1045
  await recordAudit(dataDir, "remote.check", "denied", { archiveId: record.id, reason: "absent", missing });
216
- return { ok: false, provider: remote.provider, bytes: 0, checkedAt, detail: `Your managed storage is missing ${missing} of this archive's ${refs.chunks.length + 1} objects. Do not trash the original — this disk may hold the only complete copy.` };
1046
+ return { ok: false, provider: remote.provider, bytes: 0, checkedAt, detail: `Your managed storage is missing ${missing} of this archive's ${refs.chunks.length + 1} objects. Do not reclaim the original — this disk may hold the only complete copy.` };
217
1047
  }
218
1048
  const bytes = refs.chunks.reduce((sum, ref) => sum + (rows.get(ref) ?? 0), 0);
219
1049
  if (bytes !== remote.bytes) {
220
1050
  await recordAudit(dataDir, "remote.check", "denied", { archiveId: record.id, reason: "size", stored: bytes });
221
- return { ok: false, provider: remote.provider, bytes, checkedAt, detail: `Your storage reports ${bytes} chunk bytes; ${remote.bytes} were verified. Do not trash the original.` };
1051
+ return { ok: false, provider: remote.provider, bytes, checkedAt, detail: `Your storage reports ${bytes} chunk bytes; ${remote.bytes} were verified. Do not reclaim the original.` };
222
1052
  }
223
1053
  await recordAudit(dataDir, "remote.check", "allowed", { archiveId: record.id, bytes, chunks: refs.chunks.length });
224
1054
  return { ok: true, provider: remote.provider, bytes, checkedAt, detail: `Your managed storage confirmed all ${refs.chunks.length} chunks and the sidecar at the recorded size, just now.` };
@@ -226,49 +1056,98 @@ async function managedCopyCheck(dataDir, record, rows, checkedAt) {
226
1056
  const stored = rows.get(record.id);
227
1057
  if (stored === undefined) {
228
1058
  await recordAudit(dataDir, "remote.check", "denied", { archiveId: record.id, reason: "absent" });
229
- return { ok: false, provider: remote.provider, bytes: 0, checkedAt, detail: "Your managed storage no longer lists this archive. Do not trash the original — this disk may hold the only copy." };
1059
+ return { ok: false, provider: remote.provider, bytes: 0, checkedAt, detail: "Your managed storage no longer lists this archive. Do not reclaim the original — this disk may hold the only copy." };
230
1060
  }
231
1061
  if (stored !== remote.bytes) {
232
1062
  await recordAudit(dataDir, "remote.check", "denied", { archiveId: record.id, reason: "size", stored });
233
- return { ok: false, provider: remote.provider, bytes: stored, checkedAt, detail: `Your storage reports ${stored} bytes; ${remote.bytes} were verified. Do not trash the original.` };
1063
+ return { ok: false, provider: remote.provider, bytes: stored, checkedAt, detail: `Your storage reports ${stored} bytes; ${remote.bytes} were verified. Do not reclaim the original.` };
234
1064
  }
235
1065
  await recordAudit(dataDir, "remote.check", "allowed", { archiveId: record.id, bytes: stored });
236
1066
  return { ok: true, provider: remote.provider, bytes: stored, checkedAt, detail: "Your managed storage confirmed it holds this archive at the recorded size, just now." };
237
1067
  }
238
- export async function checkRemoteCopy(dataDir, archiveId, client) {
1068
+ export async function checkRemoteCopy(dataDir, archiveId, client, copyId) {
239
1069
  const record = (await listArchives(dataDir)).find((item) => item.id === archiveId);
240
1070
  if (!record)
241
1071
  fail("archive_not_found", `Archive not found: ${archiveId}`, { archiveId });
242
1072
  if (!isV2(record))
243
1073
  fail("invalid_argument", `Archive ${archiveId} uses format v1. Run: sealkeep migrate`, { archiveId });
244
- const remote = record.remote;
1074
+ const remote = copyId
1075
+ ? archiveCopies(record).find((copy) => copy.copyId === copyId)
1076
+ : archiveCopies(record).find((copy) => copy.verifiedAt);
245
1077
  if (!remote?.verifiedAt)
246
1078
  fail("invalid_argument", "This archive has no verified cloud copy to check — upload it first.", { archiveId });
247
1079
  const checkedAt = new Date().toISOString();
248
1080
  if (remote.provider === "vaultline" && !client) {
249
1081
  const { listCloudArchives } = await import("./cloud.js");
250
1082
  const rows = new Map((await listCloudArchives(dataDir)).map((row) => [row.vault_ref, Number(row.bytes) || 0]));
251
- return managedCopyCheck(dataDir, record, rows, checkedAt);
1083
+ return managedCopyCheck(dataDir, record, remote, rows, checkedAt);
252
1084
  }
253
- const config = await readConfig(dataDir);
254
- const target = config.remoteStorage;
255
- if (!target)
256
- fail("storage_not_configured", "This vault has no storage configured, so the copy cannot be checked");
257
- const resolved = client ?? await uploadClientFromStore(dataDir, config.vaultId, target, {}, undefined);
258
- const lease = createActiveLease(target, {
259
- archiveId: record.id, ciphertextSha256: record.cipher.ciphertextSha256, bytes: record.cipher.storedBytes
260
- }, { ...process.env, SEALKEEP_ENABLE_SIGNER: "1" });
1085
+ const named = await configuredTargetForRemote(dataDir, remote);
1086
+ if (named.provider === "vaultline" || !named.bucket) {
1087
+ fail("storage_not_configured", `The ${remote.provider} destination holding this copy is not connected on this machine, so it cannot be checked.`);
1088
+ }
1089
+ const derivedPrefix = remote.objectKey.split("/").slice(0, -1).join("/");
1090
+ const target = {
1091
+ provider: named.provider,
1092
+ bucket: named.bucket,
1093
+ prefix: named.prefix ?? (derivedPrefix || "sealkeep"),
1094
+ ...(named.region ? { region: named.region } : {}),
1095
+ };
1096
+ const connection = client ? null : await ownBucketConnection(dataDir, named);
1097
+ const resolved = client ?? await uploadClientFromStore(dataDir, connection.credentialId, target, connection.overrides, undefined);
1098
+ if (remote.layout?.kind === "chunks") {
1099
+ const probing = resolved;
1100
+ if (typeof probing.headObject !== "function")
1101
+ fail("provider_unsupported", `${remote.provider} cannot probe chunk folders with this build`);
1102
+ if (remote.layout.chunkCount !== record.envelope.chunks.length) {
1103
+ return { ok: false, provider: remote.provider, bytes: 0, checkedAt, detail: `The record names ${record.envelope.chunks.length} chunks but this copy claims ${remote.layout.chunkCount}. Do not reclaim the original.` };
1104
+ }
1105
+ let bytes = 0;
1106
+ const problems = [];
1107
+ const { chunkObjectName, ENVELOPE_OBJECT } = await import("./chunk-store.js");
1108
+ for (let index = 0; index < record.envelope.chunks.length; index += 1) {
1109
+ const expected = record.envelope.chunks[index].storedBytes;
1110
+ const head = await probing.headObject(`${remote.objectKey}/${chunkObjectName(index)}`);
1111
+ if (!head.exists)
1112
+ problems.push(`chunk ${index} is missing`);
1113
+ else {
1114
+ bytes += head.bytes;
1115
+ if (head.bytes !== expected)
1116
+ problems.push(`chunk ${index} is ${head.bytes} bytes; ${expected} were verified`);
1117
+ }
1118
+ if (problems.length >= 3)
1119
+ break;
1120
+ }
1121
+ if (problems.length < 3) {
1122
+ const sidecar = await probing.headObject(`${remote.objectKey}/${ENVELOPE_OBJECT}`);
1123
+ if (!sidecar.exists)
1124
+ problems.push("the envelope sidecar is missing");
1125
+ else if (sidecar.bytes <= 0)
1126
+ problems.push("the envelope sidecar is empty");
1127
+ }
1128
+ if (bytes !== remote.bytes)
1129
+ problems.push(`the chunks total ${bytes} bytes; ${remote.bytes} were verified`);
1130
+ if (problems.length > 0) {
1131
+ await recordAudit(dataDir, "remote.check", "denied", { archiveId, reason: "chunk-folder", problems: problems.length, stored: bytes });
1132
+ return { ok: false, provider: remote.provider, bytes, checkedAt, detail: `${problems.join("; ")}. Do not reclaim the original — this disk may hold the only complete copy.` };
1133
+ }
1134
+ await recordAudit(dataDir, "remote.check", "allowed", { archiveId, bytes, chunks: remote.layout.chunkCount, sidecar: true });
1135
+ return { ok: true, provider: remote.provider, bytes, checkedAt, detail: `The bucket confirmed all ${remote.layout.chunkCount} chunks and the envelope sidecar at the recorded sizes, just now.` };
1136
+ }
1137
+ const lease = { ...createActiveLease(target, {
1138
+ archiveId: record.id, ciphertextSha256: record.cipher.ciphertextSha256, bytes: record.cipher.storedBytes
1139
+ }, { ...process.env, SEALKEEP_ENABLE_SIGNER: "1" }), objectKey: remote.objectKey };
261
1140
  const probing = resolved;
262
1141
  if (typeof probing.head !== "function")
263
- fail("provider_unsupported", `${target.provider} cannot be probed by this build`);
1142
+ fail("provider_unsupported", `${remote.provider} cannot be probed by this build`);
264
1143
  const head = await probing.head(lease);
265
1144
  if (!head.exists) {
266
1145
  await recordAudit(dataDir, "remote.check", "denied", { archiveId, reason: "absent" });
267
- return { ok: false, provider: remote.provider, bytes: 0, checkedAt, detail: "The bucket no longer holds this object. Do not trash the original — this disk may hold the only copy." };
1146
+ return { ok: false, provider: remote.provider, bytes: 0, checkedAt, detail: "The bucket no longer holds this object. Do not reclaim the original — this disk may hold the only copy." };
268
1147
  }
269
1148
  if (head.bytes !== remote.bytes) {
270
1149
  await recordAudit(dataDir, "remote.check", "denied", { archiveId, reason: "size", stored: head.bytes });
271
- return { ok: false, provider: remote.provider, bytes: head.bytes, checkedAt, detail: `The bucket holds ${head.bytes} bytes; ${remote.bytes} were verified. Do not trash the original.` };
1150
+ return { ok: false, provider: remote.provider, bytes: head.bytes, checkedAt, detail: `The bucket holds ${head.bytes} bytes; ${remote.bytes} were verified. Do not reclaim the original.` };
272
1151
  }
273
1152
  await recordAudit(dataDir, "remote.check", "allowed", { archiveId, bytes: head.bytes });
274
1153
  return { ok: true, provider: remote.provider, bytes: head.bytes, checkedAt, detail: "The bucket answered just now and holds this archive at the recorded size." };
@@ -283,7 +1162,7 @@ export async function checkRemoteCopyMany(dataDir, ids, client) {
283
1162
  const wanted = ids.map((id) => archives.find((item) => item.id === id));
284
1163
  const checkedAt = new Date().toISOString();
285
1164
  const results = [];
286
- const managedIds = wanted.filter((record) => !!record && isV2(record) && record.remote?.provider === "vaultline").map((record) => record.id);
1165
+ const managedIds = wanted.filter((record) => !!record && isV2(record) && archiveCopies(record).some((copy) => copy.verifiedAt && copy.provider === "vaultline")).map((record) => record.id);
287
1166
  let managedRows = null;
288
1167
  if (managedIds.length > 0 && !client) {
289
1168
  const { listCloudArchives } = await import("./cloud.js");
@@ -291,22 +1170,718 @@ export async function checkRemoteCopyMany(dataDir, ids, client) {
291
1170
  }
292
1171
  for (const id of ids) {
293
1172
  const record = archives.find((item) => item.id === id);
294
- if (!record || !isV2(record) || !record.remote?.verifiedAt) {
1173
+ if (!record || !isV2(record)) {
295
1174
  results.push({ ok: false, provider: "none", bytes: 0, checkedAt, detail: "No verified cloud copy on record for this archive." });
296
1175
  continue;
297
1176
  }
298
- if (record.remote.provider === "vaultline" && managedRows) {
299
- results.push(await managedCopyCheck(dataDir, record, managedRows, checkedAt));
1177
+ const copies = archiveCopies(record).filter((copy) => copy.verifiedAt);
1178
+ if (copies.length === 0) {
1179
+ results.push({ ok: false, provider: "none", bytes: 0, checkedAt, detail: "No verified cloud copy on record for this archive." });
300
1180
  continue;
301
1181
  }
1182
+ const checks = [];
1183
+ for (const copy of copies) {
1184
+ try {
1185
+ if (copy.provider === "vaultline" && managedRows && !client) {
1186
+ checks.push(await managedCopyCheck(dataDir, record, copy, managedRows, checkedAt));
1187
+ }
1188
+ else {
1189
+ checks.push(await checkRemoteCopy(dataDir, id, client, copy.copyId));
1190
+ }
1191
+ }
1192
+ catch (error) {
1193
+ checks.push({ ok: false, provider: copy.provider, bytes: 0, checkedAt, detail: error instanceof Error ? error.message : "the check itself failed" });
1194
+ }
1195
+ }
1196
+ const failed = checks.find((check) => !check.ok);
1197
+ results.push(failed ?? {
1198
+ ok: true,
1199
+ provider: checks.length === 1 ? checks[0].provider : "multiple",
1200
+ bytes: checks.reduce((sum, check) => sum + check.bytes, 0),
1201
+ checkedAt,
1202
+ detail: checks.length === 1 ? checks[0].detail : `All ${checks.length} stored copies answered at their recorded sizes, just now.`,
1203
+ });
1204
+ }
1205
+ return results;
1206
+ }
1207
+ function cancellationError(signal) {
1208
+ if (signal?.reason instanceof Error)
1209
+ return signal.reason;
1210
+ const error = new Error("The archive materialisation was cancelled");
1211
+ error.name = "AbortError";
1212
+ return error;
1213
+ }
1214
+ const DEFAULT_SCRATCH_WAIT_MS = 5_000;
1215
+ function scratchBusyError(waitMs) {
1216
+ return new SealkeepError("scratch_space_busy", `Another archive operation is using this disk's temporary workspace. Sealkeep waited ${Math.max(0, Math.round(waitMs / 100) / 10)} seconds without taking extra space; try again after that operation finishes.`, { retryable: true, waitedMs: waitMs });
1217
+ }
1218
+ function waitForScratchLease(signal, delayMs) {
1219
+ if (signal?.aborted)
1220
+ return Promise.reject(cancellationError(signal));
1221
+ return new Promise((resolve, reject) => {
1222
+ const finish = () => {
1223
+ if (signal)
1224
+ signal.removeEventListener("abort", abort);
1225
+ resolve();
1226
+ };
1227
+ const timer = setTimeout(finish, Math.max(1, delayMs));
1228
+ const abort = () => {
1229
+ clearTimeout(timer);
1230
+ signal?.removeEventListener("abort", abort);
1231
+ reject(cancellationError(signal));
1232
+ };
1233
+ signal?.addEventListener("abort", abort, { once: true });
1234
+ });
1235
+ }
1236
+ /**
1237
+ * Cleanup is secondary to the operation that made cleanup necessary. Preserve
1238
+ * the original Error object (and therefore a SealkeepError's code/details or
1239
+ * an AbortSignal's exact reason) while making a failed scratch release visible
1240
+ * to logs and callers. Frozen/non-Error throwables fall back to AggregateError,
1241
+ * which still retains both original values in order.
1242
+ */
1243
+ function withScratchReleaseFailure(primary, releaseError) {
1244
+ const releaseMessage = releaseError instanceof Error ? releaseError.message : "unknown scratch release failure";
1245
+ const detail = `releasing the archive scratch reservation also failed (${releaseMessage})`;
1246
+ if (primary instanceof Error) {
1247
+ try {
1248
+ Object.defineProperty(primary, "scratchReleaseError", {
1249
+ value: releaseError,
1250
+ configurable: true,
1251
+ enumerable: false,
1252
+ });
1253
+ primary.message = `${primary.message}; ${detail}`;
1254
+ return primary;
1255
+ }
1256
+ catch { /* a frozen/custom Error is represented by the aggregate below */ }
1257
+ }
1258
+ const primaryMessage = primary instanceof Error ? primary.message : String(primary);
1259
+ return new AggregateError([primary, releaseError], `${primaryMessage}; ${detail}`, { cause: primary });
1260
+ }
1261
+ /**
1262
+ * One complete fetched archive at a time in this process.
1263
+ *
1264
+ * Historical indexing now consumes chunk-independent archives directly from
1265
+ * their authenticated provider stream and never enters this gate. Operations
1266
+ * that genuinely require a complete file (restore, share, or provider bridge)
1267
+ * may still overlap across local servers. The permit is held until the verified
1268
+ * temporary file is released, not merely until the download ends: measured free
1269
+ * space therefore reflects the previous archive's deletion before the next
1270
+ * check. Local archives never enter this gate and retain their parallel path.
1271
+ */
1272
+ class ScratchMaterialisationGate {
1273
+ active = false;
1274
+ waiting = [];
1275
+ acquire(signal, deadline, waitMs) {
1276
+ if (signal?.aborted)
1277
+ return Promise.reject(cancellationError(signal));
1278
+ if (Date.now() >= deadline)
1279
+ return Promise.reject(scratchBusyError(waitMs));
1280
+ return new Promise((grant, reject) => {
1281
+ const waiter = { signal, grant, reject };
1282
+ const withdraw = (error) => {
1283
+ const at = this.waiting.indexOf(waiter);
1284
+ if (at >= 0)
1285
+ this.waiting.splice(at, 1);
1286
+ if (waiter.timer)
1287
+ clearTimeout(waiter.timer);
1288
+ signal?.removeEventListener("abort", waiter.onAbort);
1289
+ reject(error);
1290
+ };
1291
+ if (signal) {
1292
+ waiter.onAbort = () => withdraw(cancellationError(signal));
1293
+ signal.addEventListener("abort", waiter.onAbort, { once: true });
1294
+ }
1295
+ waiter.timer = setTimeout(() => withdraw(scratchBusyError(waitMs)), Math.max(1, deadline - Date.now()));
1296
+ if (!this.active)
1297
+ this.activate(waiter);
1298
+ else
1299
+ this.waiting.push(waiter);
1300
+ });
1301
+ }
1302
+ activate(waiter) {
1303
+ if (waiter.signal?.aborted) {
1304
+ waiter.onAbort?.();
1305
+ this.advance();
1306
+ return;
1307
+ }
1308
+ this.active = true;
1309
+ if (waiter.timer)
1310
+ clearTimeout(waiter.timer);
1311
+ if (waiter.signal && waiter.onAbort)
1312
+ waiter.signal.removeEventListener("abort", waiter.onAbort);
1313
+ let released = false;
1314
+ waiter.grant(() => {
1315
+ if (released)
1316
+ return;
1317
+ released = true;
1318
+ this.active = false;
1319
+ this.advance();
1320
+ });
1321
+ }
1322
+ advance() {
1323
+ if (this.active)
1324
+ return;
1325
+ const next = this.waiting.shift();
1326
+ if (next)
1327
+ this.activate(next);
1328
+ }
1329
+ }
1330
+ const scratchMaterialisations = new ScratchMaterialisationGate();
1331
+ const SCRATCH_LEASE_STALE_MS = 30 * 60_000;
1332
+ const scratchLeaseOwnerPath = (leasePath) => join(leasePath, "owner.json");
1333
+ /**
1334
+ * Processes from different vaults can still spend the same filesystem's free
1335
+ * bytes. Key the well-known lease by the scratch directory's device id in a
1336
+ * per-user temp namespace; fall back to this vault's trusted runtime directory
1337
+ * only on platforms where that identity cannot be established.
1338
+ */
1339
+ export async function ensurePrivateScratchCoordination(baseDir, user, expectedUid = typeof process.getuid === "function" ? process.getuid() : undefined) {
1340
+ const namespace = join(baseDir, `sealkeep-${user}`);
1341
+ const coordination = join(namespace, "scratch-volume-leases");
1342
+ for (const path of [namespace, coordination]) {
302
1343
  try {
303
- results.push(await checkRemoteCopy(dataDir, id, client));
1344
+ await mkdir(path, { mode: 0o700 });
304
1345
  }
305
1346
  catch (error) {
306
- results.push({ ok: false, provider: record.remote.provider, bytes: 0, checkedAt, detail: error instanceof Error ? error.message : "the check itself failed" });
1347
+ if (error.code !== "EEXIST")
1348
+ throw error;
1349
+ }
1350
+ const entry = await lstat(path);
1351
+ if (entry.isSymbolicLink() || !entry.isDirectory()) {
1352
+ return fail("internal", `Sealkeep refused an unsafe archive scratch coordination path (${path} is not a real directory).`);
1353
+ }
1354
+ if (expectedUid !== undefined && entry.uid !== expectedUid) {
1355
+ return fail("internal", `Sealkeep refused an archive scratch coordination directory not owned by this user (${path}).`);
1356
+ }
1357
+ if ((entry.mode & 0o777) !== 0o700) {
1358
+ return fail("internal", `Sealkeep refused an archive scratch coordination directory without private 0700 permissions (${path}).`);
307
1359
  }
308
1360
  }
309
- return results;
1361
+ return coordination;
1362
+ }
1363
+ async function scratchLeasePath(dataDir, spaceRoot) {
1364
+ try {
1365
+ const device = (await stat(spaceRoot)).dev;
1366
+ const user = typeof process.getuid === "function" ? String(process.getuid()) : "user";
1367
+ // Never let a predictable /tmp pathname be redirected through a symlink
1368
+ // or pre-created by another local account. Each component below the OS
1369
+ // temp root is a real, current-user-owned 0700 directory before a ticket
1370
+ // or lease token is written into it.
1371
+ const coordination = await ensurePrivateScratchCoordination(tmpdir(), user);
1372
+ return join(coordination, `device-${device}.lock`);
1373
+ }
1374
+ catch (error) {
1375
+ // A per-vault fallback is not safe: two vaults on the same nearly-full
1376
+ // volume would each believe they exclusively reserved the remaining disk.
1377
+ // Fail closed when the shared volume identity/coordination directory is
1378
+ // unavailable; the original provider bytes remain untouched.
1379
+ return fail("internal", `Sealkeep could not reserve this volume's shared archive workspace: ${error instanceof Error ? error.message : "coordination unavailable"}`, { dataDir });
1380
+ }
1381
+ }
1382
+ function parseScratchLeaseOwner(raw) {
1383
+ if (!raw)
1384
+ return null;
1385
+ try {
1386
+ const value = JSON.parse(raw);
1387
+ if (value.version !== 1 || !Number.isSafeInteger(value.pid) || Number(value.pid) < 1
1388
+ || typeof value.processIdentity !== "string" || value.processIdentity.length < 1 || value.processIdentity.length > 512
1389
+ || typeof value.nonce !== "string" || !/^[0-9a-f-]{36}$/i.test(value.nonce)
1390
+ || typeof value.acquiredAt !== "string")
1391
+ return null;
1392
+ return value;
1393
+ }
1394
+ catch {
1395
+ return null;
1396
+ }
1397
+ }
1398
+ /**
1399
+ * Separate local servers can perform share/restore/transfer work at once, and
1400
+ * an older index child may still own the v1 lease during an upgrade. The
1401
+ * in-memory gate above cannot make their `statfs` checks atomic. This volume
1402
+ * lease spans download plus consumer use across processes; a dead process
1403
+ * generation is reclaimable, while an unidentifiable live owner fails closed
1404
+ * and keeps its possible scratch bytes reserved.
1405
+ */
1406
+ async function acquireCrossProcessScratchLease(dataDir, spaceRoot, signal, deadline, waitMs) {
1407
+ if (signal?.aborted)
1408
+ throw cancellationError(signal);
1409
+ const identityBeforeDeadline = async (pid) => {
1410
+ const remaining = deadline - Date.now();
1411
+ if (remaining <= 0)
1412
+ throw scratchBusyError(waitMs);
1413
+ let timer;
1414
+ try {
1415
+ return await Promise.race([
1416
+ daemonProcessIdentity(pid),
1417
+ new Promise((resolve) => {
1418
+ timer = setTimeout(() => resolve({ state: "unknown" }), remaining);
1419
+ }),
1420
+ ]);
1421
+ }
1422
+ finally {
1423
+ if (timer)
1424
+ clearTimeout(timer);
1425
+ }
1426
+ };
1427
+ const self = await identityBeforeDeadline(process.pid);
1428
+ if (self.state !== "identified") {
1429
+ return fail("internal", "Sealkeep could not verify this process generation before reserving archive scratch space");
1430
+ }
1431
+ const leasePath = await scratchLeasePath(dataDir, spaceRoot);
1432
+ const runtimeDir = dirname(leasePath);
1433
+ await mkdir(runtimeDir, { recursive: true, mode: 0o700 });
1434
+ const owner = {
1435
+ version: 1,
1436
+ pid: process.pid,
1437
+ processIdentity: self.identity,
1438
+ nonce: randomUUID(),
1439
+ acquiredAt: new Date().toISOString(),
1440
+ };
1441
+ const token = `${JSON.stringify(owner)}\n`;
1442
+ let acquired;
1443
+ // A tiny FIFO ticket stays outside the long-held lease. Unlike repeated
1444
+ // rename races, lexicographic admission cannot indefinitely starve one
1445
+ // process behind newer contenders. The content carries the same process-
1446
+ // generation proof as the lease, so a crashed waiter can be removed safely.
1447
+ const ticketPrefix = `${basename(leasePath)}.ticket-`;
1448
+ const ticketName = `${ticketPrefix}${String(Date.now()).padStart(16, "0")}-${String(process.pid).padStart(10, "0")}-${owner.nonce}.json`;
1449
+ const ticketPath = join(runtimeDir, ticketName);
1450
+ const ticketPending = `${ticketPath}.pending`;
1451
+ try {
1452
+ await writeFile(ticketPending, token, { flag: "wx", mode: 0o600 });
1453
+ await rename(ticketPending, ticketPath);
1454
+ }
1455
+ catch (error) {
1456
+ await unlink(ticketPending).catch(() => undefined);
1457
+ throw error;
1458
+ }
1459
+ const liveTickets = async () => {
1460
+ // Tickets after ours cannot delay us. Avoid process-generation probes for
1461
+ // those newer waiters, and stop the stale-ticket scan at the same deadline
1462
+ // as acquisition so directory litter cannot stretch a five-second bound.
1463
+ const names = (await readdir(runtimeDir))
1464
+ .filter((name) => name.startsWith(ticketPrefix) && name.endsWith(".json") && name <= ticketName)
1465
+ .sort();
1466
+ const live = [];
1467
+ for (const name of names) {
1468
+ if (Date.now() >= deadline)
1469
+ throw scratchBusyError(waitMs);
1470
+ const path = join(runtimeDir, name);
1471
+ const [raw, entry] = await Promise.all([
1472
+ readFile(path, "utf8").catch(() => null),
1473
+ stat(path).catch(() => null),
1474
+ ]);
1475
+ if (!entry)
1476
+ continue;
1477
+ if (name === ticketName && raw === token) {
1478
+ live.push(name);
1479
+ continue;
1480
+ }
1481
+ const ticketOwner = parseScratchLeaseOwner(raw);
1482
+ let abandoned = false;
1483
+ if (ticketOwner) {
1484
+ const generation = await identityBeforeDeadline(ticketOwner.pid);
1485
+ abandoned = classifyDaemonProcessGeneration(generation, ticketOwner.processIdentity) === "stale";
1486
+ }
1487
+ else if (Date.now() - entry.mtimeMs > SCRATCH_LEASE_STALE_MS) {
1488
+ abandoned = true;
1489
+ }
1490
+ if (abandoned) {
1491
+ // The user-private coordination directory prevents hostile replacement,
1492
+ // but still compare the complete token so two legitimate rescuers never
1493
+ // unlink a newer ticket that reused the same pathname.
1494
+ if (await readFile(path, "utf8").then((current) => current === raw).catch(() => false)) {
1495
+ await unlink(path).catch(() => undefined);
1496
+ }
1497
+ continue;
1498
+ }
1499
+ live.push(name);
1500
+ }
1501
+ return live;
1502
+ };
1503
+ try {
1504
+ for (;;) {
1505
+ if (signal?.aborted)
1506
+ throw cancellationError(signal);
1507
+ if (Date.now() >= deadline)
1508
+ throw scratchBusyError(waitMs);
1509
+ const tickets = await liveTickets();
1510
+ if (tickets[0] !== ticketName) {
1511
+ await waitForScratchLease(signal, Math.min(50, Math.max(1, deadline - Date.now())));
1512
+ continue;
1513
+ }
1514
+ const staging = `${leasePath}.${owner.nonce}.pending`;
1515
+ try {
1516
+ await mkdir(staging, { mode: 0o700 });
1517
+ await writeFile(scratchLeaseOwnerPath(staging), token, { flag: "wx", mode: 0o600 });
1518
+ const candidate = await stat(staging);
1519
+ await rename(staging, leasePath);
1520
+ acquired = { dev: candidate.dev, ino: candidate.ino, birthtimeMs: candidate.birthtimeMs };
1521
+ break;
1522
+ }
1523
+ catch (error) {
1524
+ await unlink(scratchLeaseOwnerPath(staging)).catch(() => undefined);
1525
+ await rmdir(staging).catch(() => undefined);
1526
+ const code = error.code;
1527
+ const leaseExists = await stat(leasePath).then((entry) => entry.isDirectory()).catch(() => false);
1528
+ // A legitimate owner can release between our failed publish and this
1529
+ // observation. Its now-absent directory is a retry opportunity, not an
1530
+ // archive-transfer error. Never infer the same from a permission error.
1531
+ if (!leaseExists && (code === "EEXIST" || code === "ENOTEMPTY"))
1532
+ continue;
1533
+ if (!leaseExists || (code !== "EEXIST" && code !== "ENOTEMPTY" && code !== "EPERM"))
1534
+ throw error;
1535
+ const [held, raw] = await Promise.all([
1536
+ stat(leasePath).catch(() => null),
1537
+ readFile(scratchLeaseOwnerPath(leasePath), "utf8").catch(() => null),
1538
+ ]);
1539
+ const currentOwner = parseScratchLeaseOwner(raw);
1540
+ let abandoned = false;
1541
+ if (currentOwner) {
1542
+ const generation = await identityBeforeDeadline(currentOwner.pid);
1543
+ abandoned = classifyDaemonProcessGeneration(generation, currentOwner.processIdentity) === "stale";
1544
+ }
1545
+ else if (held && Date.now() - held.mtimeMs > SCRATCH_LEASE_STALE_MS) {
1546
+ // Complete lock directories are published by rename, so malformed
1547
+ // content is never a normal acquisition window. Keep it for a generous
1548
+ // transition period before treating old/corrupt state as abandoned.
1549
+ abandoned = true;
1550
+ }
1551
+ if (abandoned && held) {
1552
+ const fingerprint = createHash("sha256")
1553
+ .update(`${held.dev}:${held.ino}:${held.birthtimeMs}:${raw ?? ""}`)
1554
+ .digest("hex")
1555
+ .slice(0, 20);
1556
+ const quarantine = `${leasePath}.stale-${fingerprint}`;
1557
+ const [again, currentRaw] = await Promise.all([
1558
+ stat(leasePath).catch(() => null),
1559
+ readFile(scratchLeaseOwnerPath(leasePath), "utf8").catch(() => null),
1560
+ ]);
1561
+ const same = !!again && again.dev === held.dev && again.ino === held.ino
1562
+ && again.birthtimeMs === held.birthtimeMs && currentRaw === raw;
1563
+ if (same) {
1564
+ try {
1565
+ await rename(leasePath, quarantine);
1566
+ }
1567
+ catch (moveError) {
1568
+ const moveCode = moveError.code;
1569
+ if (moveCode !== "ENOENT" && moveCode !== "EEXIST" && moveCode !== "ENOTEMPTY" && moveCode !== "EPERM")
1570
+ throw moveError;
1571
+ }
1572
+ }
1573
+ // Deliberately retain the non-empty quarantine. If two rescuers saw
1574
+ // this dead directory, the first can move it and a new owner can then
1575
+ // acquire the public name before the second reaches rename(). The
1576
+ // deterministic occupied destination makes that delayed rename fail
1577
+ // instead of moving the successor's live lease out of the way.
1578
+ continue;
1579
+ }
1580
+ await waitForScratchLease(signal, Math.min(50, Math.max(1, deadline - Date.now())));
1581
+ }
1582
+ }
1583
+ }
1584
+ finally {
1585
+ await unlink(ticketPath).catch(() => undefined);
1586
+ await unlink(ticketPending).catch(() => undefined);
1587
+ }
1588
+ if (!acquired)
1589
+ return fail("internal", "Sealkeep lost its archive scratch reservation while acquiring it");
1590
+ return async () => {
1591
+ const absentOnly = async (operation) => {
1592
+ try {
1593
+ return await operation();
1594
+ }
1595
+ catch (error) {
1596
+ if (error.code === "ENOENT")
1597
+ return null;
1598
+ throw error;
1599
+ }
1600
+ };
1601
+ const [held, raw] = await Promise.all([
1602
+ absentOnly(() => stat(leasePath)),
1603
+ absentOnly(() => readFile(scratchLeaseOwnerPath(leasePath), "utf8")),
1604
+ ]);
1605
+ if (!held || held.dev !== acquired.dev || held.ino !== acquired.ino
1606
+ || held.birthtimeMs !== acquired.birthtimeMs || raw !== token)
1607
+ return;
1608
+ const released = `${leasePath}.released-${owner.nonce}`;
1609
+ try {
1610
+ await rename(leasePath, released);
1611
+ }
1612
+ catch (error) {
1613
+ if (error.code === "ENOENT")
1614
+ return;
1615
+ throw error;
1616
+ }
1617
+ await unlink(scratchLeaseOwnerPath(released)).catch(() => undefined);
1618
+ await rmdir(released).catch(() => undefined);
1619
+ };
1620
+ }
1621
+ async function acquireScratchMaterialisationPermit(dataDir, spaceRoot, signal, scratchWaitMs = DEFAULT_SCRATCH_WAIT_MS) {
1622
+ const waitMs = Math.max(1, Math.min(5 * 60_000, Math.ceil(scratchWaitMs)));
1623
+ const deadline = Date.now() + waitMs;
1624
+ const releaseLocal = await scratchMaterialisations.acquire(signal, deadline, waitMs);
1625
+ try {
1626
+ const releaseCrossProcess = await acquireCrossProcessScratchLease(dataDir, spaceRoot, signal, deadline, waitMs);
1627
+ let localReleased = false;
1628
+ let releaseComplete = false;
1629
+ let releaseInFlight;
1630
+ return () => {
1631
+ if (releaseComplete)
1632
+ return Promise.resolve();
1633
+ if (releaseInFlight)
1634
+ return releaseInFlight;
1635
+ const attempt = (async () => {
1636
+ try {
1637
+ await releaseCrossProcess();
1638
+ releaseComplete = true;
1639
+ }
1640
+ finally {
1641
+ // A failed cross-process rename must not monopolise this process's
1642
+ // FIFO lane. Its exact-owner callback remains retryable, while this
1643
+ // local release happens once and only once.
1644
+ if (!localReleased) {
1645
+ localReleased = true;
1646
+ releaseLocal();
1647
+ }
1648
+ }
1649
+ })();
1650
+ releaseInFlight = attempt;
1651
+ attempt.then(() => { if (releaseInFlight === attempt)
1652
+ releaseInFlight = undefined; }, () => { if (releaseInFlight === attempt)
1653
+ releaseInFlight = undefined; });
1654
+ return attempt;
1655
+ };
1656
+ }
1657
+ catch (error) {
1658
+ releaseLocal();
1659
+ throw error;
1660
+ }
1661
+ }
1662
+ /**
1663
+ * Restores the sealed archive blob itself to this machine without decrypting
1664
+ * the session. This is the safe first half of a remote-to-remote move: once a
1665
+ * verified local blob exists, the ordinary target upload can copy it anywhere
1666
+ * without buffering two providers through the browser or risking the source.
1667
+ */
1668
+ export async function restoreArchiveLocal(dataDir, archiveId, options = {}) {
1669
+ const config = await readConfig(dataDir);
1670
+ const record = (await listArchives(dataDir)).find((item) => item.id === archiveId);
1671
+ if (!record)
1672
+ fail("archive_not_found", `Archive not found: ${archiveId}`, { archiveId });
1673
+ if (!isV2(record))
1674
+ fail("invalid_argument", `Archive ${archiveId} uses format v1. Run: sealkeep migrate`, { archiveId });
1675
+ const present = await stat(record.objectPath).then((entry) => entry.isFile()).catch(() => false);
1676
+ if (present) {
1677
+ const { hashFileRange } = await import("../packages/sealkeep-crypto/src/index.js");
1678
+ const measured = await hashFileRange(record.objectPath, 0, undefined, options.onProgress, options.signal);
1679
+ if (measured.sha256 !== record.cipher.ciphertextSha256) {
1680
+ fail("ciphertext_integrity_failed", `A file already exists at ${record.objectPath}, but it is not archive ${archiveId}. It was left untouched.`, { archiveId, path: record.objectPath });
1681
+ }
1682
+ if (record.offloaded) {
1683
+ await mutateArchiveRecord(config.storage.root, archiveId, (current) => {
1684
+ if (!isV2(current))
1685
+ return current;
1686
+ const { offloaded: _offloaded, ...withLocalCopy } = current;
1687
+ return withLocalCopy;
1688
+ });
1689
+ }
1690
+ const bytes = measured.bytes;
1691
+ return { restored: false, alreadyPresent: true, bytes, path: record.objectPath };
1692
+ }
1693
+ const copies = archiveCopies(record);
1694
+ const chosen = options.copyId
1695
+ ? copies.find((copy) => copy.copyId === options.copyId)
1696
+ : copies.find((copy) => copy.verifiedAt);
1697
+ if (!chosen) {
1698
+ fail("archive_not_found", options.copyId
1699
+ ? `Archive ${archiveId} has no stored copy ${options.copyId}`
1700
+ : `Archive ${archiveId} has no verified stored copy to restore`, { archiveId, copyId: options.copyId });
1701
+ }
1702
+ const { freeBytes, resolveReserveBytes } = await import("./disk.js");
1703
+ const [free, reserve] = await Promise.all([
1704
+ freeBytes(dirname(record.objectPath)),
1705
+ resolveReserveBytes(dataDir),
1706
+ ]);
1707
+ const needed = record.cipher.storedBytes;
1708
+ if (free !== undefined && free < needed + reserve) {
1709
+ const gb = (value) => `${(value / 1024 ** 3).toFixed(1)} GB`;
1710
+ fail("insufficient_disk_space", `Restoring archive ${archiveId.slice(0, 8)} needs about ${gb(needed)} plus ${gb(reserve)} of working room; this disk has ${gb(free)} free. Nothing was downloaded.`, { archiveId, free, needed, reserve });
1711
+ }
1712
+ await mkdir(dirname(record.objectPath), { recursive: true, mode: 0o700 });
1713
+ const temporary = `${record.objectPath}.${randomUUID()}.restore`;
1714
+ let writtenBytes = 0;
1715
+ try {
1716
+ // Provider bytes land directly in a private temporary file. Hashing is in
1717
+ // the same write loop, so no complete archive exists in memory and an
1718
+ // invalid response never reaches the canonical pathname.
1719
+ const measured = await writeChunksToFile(temporary, await fetchCiphertextStream(dataDir, record, options.client, chosen, options.overrides, options.signal), { mode: 0o600, expectedBytes: record.cipher.storedBytes, signal: options.signal, onProgress: options.onProgress });
1720
+ if (measured.sha256Hex !== record.cipher.ciphertextSha256 || measured.bytes !== record.cipher.storedBytes) {
1721
+ fail("ciphertext_integrity_failed", `Stored copy ${chosen.copyId} is not archive ${archiveId}; nothing was written`, { archiveId, copyId: chosen.copyId });
1722
+ }
1723
+ writtenBytes = measured.bytes;
1724
+ await rename(temporary, record.objectPath);
1725
+ await mutateArchiveRecord(config.storage.root, archiveId, (current) => {
1726
+ if (!isV2(current))
1727
+ return current;
1728
+ const { offloaded: _offloaded, ...withLocalCopy } = current;
1729
+ return withLocalCopy;
1730
+ });
1731
+ }
1732
+ catch (error) {
1733
+ await rm(temporary, { force: true }).catch(() => undefined);
1734
+ throw error;
1735
+ }
1736
+ await recordAudit(dataDir, "archive.restore_local", "allowed", { archiveId, copyId: chosen.copyId ?? null, bytes: writtenBytes }).catch(() => undefined);
1737
+ return { restored: true, alreadyPresent: false, bytes: writtenBytes, path: record.objectPath };
1738
+ }
1739
+ /**
1740
+ * Restore one logical session's complete encrypted delta chain.
1741
+ *
1742
+ * Every missing link is downloaded into a private sibling temporary file and
1743
+ * checked against its authenticated ciphertext length and digest first. Only
1744
+ * after the *whole* chain passes are the files hard-linked into their canonical
1745
+ * paths. A hard link gives us an exclusive publish operation: another process
1746
+ * cannot race this restore into overwriting an existing archive. If any publish
1747
+ * fails, every path published by this call is removed again, while all remote
1748
+ * copies and archive records remain untouched.
1749
+ *
1750
+ * Metadata is changed only after every required local file exists. A metadata
1751
+ * write failure deliberately keeps the complete verified local chain as an
1752
+ * extra safe copy and leaves all remote copies in place; it can never expose a
1753
+ * head without its base or authorize a provider delete.
1754
+ */
1755
+ export async function restoreArchiveChainLocal(dataDir, archiveId, options = {}) {
1756
+ const config = await readConfig(dataDir);
1757
+ const archives = await listArchives(dataDir);
1758
+ const record = archives.find((item) => item.id === archiveId);
1759
+ if (!record)
1760
+ fail("archive_not_found", `Archive not found: ${archiveId}`, { archiveId });
1761
+ if (!isV2(record))
1762
+ fail("invalid_argument", `Archive ${archiveId} uses format v1. Run: sealkeep migrate`, { archiveId });
1763
+ const chain = resolveDeltaChain(archives, record);
1764
+ const alreadyPresent = [];
1765
+ const missing = [];
1766
+ // Refuse conflicting local paths before making a provider request. A
1767
+ // same-name file is not evidence that a chain dependency is present.
1768
+ for (const item of chain) {
1769
+ if (!isV2(item))
1770
+ fail("invalid_argument", `Archive ${item.id} uses format v1. Run: sealkeep migrate`, { archiveId: item.id });
1771
+ const present = await stat(item.objectPath).then((entry) => entry.isFile()).catch(() => false);
1772
+ if (present) {
1773
+ const { hashFileRange } = await import("../packages/sealkeep-crypto/src/index.js");
1774
+ const measured = await hashFileRange(item.objectPath);
1775
+ if (measured.bytes !== item.cipher.storedBytes || measured.sha256 !== item.cipher.ciphertextSha256) {
1776
+ fail("ciphertext_integrity_failed", `A file already exists at ${item.objectPath}, but it is not archive ${item.id}. It was left untouched.`, {
1777
+ archiveId: item.id,
1778
+ path: item.objectPath,
1779
+ deltaDependency: item.id !== archiveId,
1780
+ });
1781
+ }
1782
+ alreadyPresent.push(item);
1783
+ continue;
1784
+ }
1785
+ const requestedCopyId = options.copyIds?.[item.id]
1786
+ ?? (item.id === archiveId ? options.copyId : undefined);
1787
+ const copies = archiveCopies(item);
1788
+ const chosen = requestedCopyId
1789
+ ? copies.find((copy) => copy.copyId === requestedCopyId && copy.verifiedAt)
1790
+ : copies.find((copy) => copy.verifiedAt);
1791
+ if (!chosen) {
1792
+ fail("archive_not_found", requestedCopyId
1793
+ ? `Archive ${item.id} no longer has verified stored copy ${requestedCopyId}; no local chain was published.`
1794
+ : `Archive ${item.id} has no verified stored copy; no local chain was published.`, {
1795
+ archiveId: item.id,
1796
+ copyId: requestedCopyId,
1797
+ deltaDependency: item.id !== archiveId,
1798
+ });
1799
+ }
1800
+ missing.push({
1801
+ record: item,
1802
+ copy: chosen,
1803
+ temporary: `${item.objectPath}.${randomUUID()}.restore`,
1804
+ });
1805
+ }
1806
+ const needed = missing.reduce((sum, item) => sum + item.record.cipher.storedBytes, 0);
1807
+ if (needed > 0) {
1808
+ const { freeBytes, resolveReserveBytes } = await import("./disk.js");
1809
+ const [free, reserve] = await Promise.all([
1810
+ freeBytes(dirname(record.objectPath)),
1811
+ resolveReserveBytes(dataDir),
1812
+ ]);
1813
+ if (free !== undefined && free < needed + reserve) {
1814
+ const gb = (value) => `${(value / 1024 ** 3).toFixed(1)} GB`;
1815
+ fail("insufficient_disk_space", `Restoring this ${chain.length}-link session chain needs about ${gb(needed)} plus ${gb(reserve)} of working room; this disk has ${gb(free)} free. Nothing was downloaded.`, { archiveId, chain: chain.map((item) => item.id), free, needed, reserve });
1816
+ }
1817
+ }
1818
+ const staged = [];
1819
+ const published = [];
1820
+ try {
1821
+ // Stage and prove every byte before publishing even the oldest base.
1822
+ for (const item of missing) {
1823
+ await mkdir(dirname(item.record.objectPath), { recursive: true, mode: 0o700 });
1824
+ const measured = await writeChunksToFile(item.temporary, await fetchCiphertextStream(dataDir, item.record, options.client, item.copy, options.overrides), { mode: 0o600, expectedBytes: item.record.cipher.storedBytes });
1825
+ if (measured.sha256Hex !== item.record.cipher.ciphertextSha256
1826
+ || measured.bytes !== item.record.cipher.storedBytes) {
1827
+ fail("ciphertext_integrity_failed", `Stored copy ${item.copy.copyId} is not archive ${item.record.id}; no local chain was published.`, {
1828
+ archiveId: item.record.id,
1829
+ copyId: item.copy.copyId,
1830
+ deltaDependency: item.record.id !== archiveId,
1831
+ });
1832
+ }
1833
+ staged.push(item);
1834
+ }
1835
+ // Hard-link publication is exclusive and cannot overwrite a path that
1836
+ // appeared after the preflight. All links live beside their destinations,
1837
+ // so this never crosses a filesystem boundary.
1838
+ for (const item of staged) {
1839
+ await hardLink(item.temporary, item.record.objectPath);
1840
+ published.push(item);
1841
+ }
1842
+ }
1843
+ catch (error) {
1844
+ for (const item of published.reverse())
1845
+ await rm(item.record.objectPath, { force: true }).catch(() => undefined);
1846
+ throw error;
1847
+ }
1848
+ finally {
1849
+ for (const item of missing)
1850
+ await rm(item.temporary, { force: true }).catch(() => undefined);
1851
+ }
1852
+ // Every link is now physically present and verified. Clear stale
1853
+ // remote-only locators without forgetting any durable remote copy.
1854
+ for (const item of chain) {
1855
+ if (!isV2(item) || !item.offloaded)
1856
+ continue;
1857
+ await mutateArchiveRecord(config.storage.root, item.id, (current) => {
1858
+ if (!isV2(current))
1859
+ return current;
1860
+ const { offloaded: _offloaded, ...withLocalCopy } = current;
1861
+ return withLocalCopy;
1862
+ });
1863
+ }
1864
+ const restoredArchiveIds = missing.map((item) => item.record.id);
1865
+ const alreadyPresentArchiveIds = alreadyPresent.map((item) => item.id);
1866
+ const totalBytes = chain.reduce((sum, item) => sum + (isV2(item) ? item.cipher.storedBytes : 0), 0);
1867
+ const restoredBytes = missing.reduce((sum, item) => sum + item.record.cipher.storedBytes, 0);
1868
+ await recordAudit(dataDir, "archive.restore_local", "allowed", {
1869
+ archiveId,
1870
+ chainLength: chain.length,
1871
+ restoredCount: restoredArchiveIds.length,
1872
+ alreadyPresentCount: alreadyPresentArchiveIds.length,
1873
+ bytes: restoredBytes,
1874
+ }).catch(() => undefined);
1875
+ return {
1876
+ restored: missing.length > 0,
1877
+ alreadyPresent: missing.length === 0,
1878
+ bytes: totalBytes,
1879
+ restoredBytes,
1880
+ path: record.objectPath,
1881
+ chain: chain.map((item) => item.id),
1882
+ restoredArchiveIds,
1883
+ alreadyPresentArchiveIds,
1884
+ };
310
1885
  }
311
1886
  /**
312
1887
  * Gives back a local path holding this archive's ciphertext, fetching it first
@@ -317,28 +1892,161 @@ export async function checkRemoteCopyMany(dataDir, ids, client) {
317
1892
  * one and removed afterwards, which keeps the disk saving intact: the bytes are
318
1893
  * on this machine only for as long as the restore takes.
319
1894
  */
320
- export async function materialiseArchive(dataDir, record, client) {
321
- if (!isV2(record) || !record.offloaded)
1895
+ export async function materialiseArchive(dataDir, record, client, options = {}) {
1896
+ if (!isV2(record) || (!record.offloaded && !options.copyId))
322
1897
  return { path: record.objectPath, release: async () => { } };
1898
+ const selected = options.copyId
1899
+ ? archiveCopies(record).find((copy) => copy.copyId === options.copyId)
1900
+ : archiveCopies(record).find((copy) => copy.provider === record.offloaded?.provider
1901
+ && copy.bucket === record.offloaded.bucket
1902
+ && copy.objectKey === record.offloaded.objectKey);
1903
+ if (!selected) {
1904
+ fail("archive_not_found", options.copyId
1905
+ ? `Archive ${record.id} has no stored copy ${options.copyId}`
1906
+ : `Archive ${record.id} has no stored copy matching its offloaded locator`, { archiveId: record.id, copyId: options.copyId });
1907
+ }
1908
+ const ownedPath = options.path;
1909
+ if (ownedPath) {
1910
+ const existing = await stat(ownedPath).then((entry) => entry.isFile()).catch(() => false);
1911
+ if (existing) {
1912
+ const { hashFileRange } = await import("../packages/sealkeep-crypto/src/index.js");
1913
+ const measured = await hashFileRange(ownedPath, 0, undefined, options.onProgress, options.signal);
1914
+ if (measured.sha256 !== record.cipher.ciphertextSha256) {
1915
+ fail("ciphertext_integrity_failed", `The transfer staging file for archive ${record.id} is not the recorded ciphertext. It was left untouched.`, { archiveId: record.id, path: ownedPath });
1916
+ }
1917
+ return {
1918
+ path: ownedPath,
1919
+ release: async () => {
1920
+ await rm(ownedPath, { force: true });
1921
+ await rm(`${ownedPath}.partial`, { force: true });
1922
+ },
1923
+ };
1924
+ }
1925
+ }
1926
+ const spaceRoot = ownedPath ? dirname(ownedPath) : options.scratchRoot ?? tmpdir();
1927
+ if (ownedPath)
1928
+ await mkdir(spaceRoot, { recursive: true, mode: 0o700 });
1929
+ // Claim the shared scratch lane BEFORE measuring free space. Holding this
1930
+ // permit through `release()` makes the measurement and the archive-sized
1931
+ // lifetime one atomic decision with respect to parent and child processes.
1932
+ const releaseScratchPermit = await acquireScratchMaterialisationPermit(dataDir, spaceRoot, options.signal, options.scratchWaitMs);
1933
+ let permitReleaseComplete = false;
1934
+ let permitReleaseInFlight;
1935
+ const relinquishPermit = () => {
1936
+ if (permitReleaseComplete)
1937
+ return Promise.resolve();
1938
+ if (permitReleaseInFlight)
1939
+ return permitReleaseInFlight;
1940
+ const attempt = (async () => { await releaseScratchPermit(); })();
1941
+ permitReleaseInFlight = attempt;
1942
+ attempt.then(() => {
1943
+ permitReleaseComplete = true;
1944
+ if (permitReleaseInFlight === attempt)
1945
+ permitReleaseInFlight = undefined;
1946
+ }, () => { if (permitReleaseInFlight === attempt)
1947
+ permitReleaseInFlight = undefined; });
1948
+ return attempt;
1949
+ };
323
1950
  // Reading an offloaded archive means writing it back to disk in full, and
324
1951
  // this runs in a loop during an index build. On a machine that offloaded
325
1952
  // BECAUSE it was short of space, that filled the disk and died on ENOSPC
326
1953
  // mid-build — the exact failure this product exists to prevent. Ask first,
327
1954
  // and refuse in words that name the archive and the shortfall.
328
- const { freeBytes } = await import("./disk.js");
329
- const free = await freeBytes(tmpdir());
330
- const needed = record.cipher.storedBytes ?? record.remote?.bytes ?? 0;
331
- const RESERVE = 512 * 1024 * 1024;
332
- if (free !== undefined && needed > 0 && free < needed + RESERVE) {
333
- const gb = (n) => `${(n / 1024 ** 3).toFixed(1)} GB`;
334
- fail("insufficient_disk_space", `Reading archive ${record.id.slice(0, 8)} needs about ${gb(needed)} of temporary space and this machine has ${gb(free)} free. Nothing was written. Free some space, or reclaim what is already archived, and try again.`, { archiveId: record.id, free, needed });
335
- }
336
- const ciphertext = await fetchCiphertext(dataDir, record, client);
337
- if (sha256(ciphertext) !== record.cipher.ciphertextSha256) {
338
- fail("ciphertext_integrity_failed", `The copy of archive ${record.id} in ${record.offloaded.bucket} is not the archive that was stored there`, { archiveId: record.id });
339
- }
340
- const scratch = await mkdtemp(join(tmpdir(), "vaultline-fetch-"));
341
- const path = join(scratch, `${record.id}.vlarchive`);
342
- await writeFile(path, ciphertext, { mode: 0o600 });
343
- return { path, release: async () => { await rm(scratch, { recursive: true, force: true }); } };
1955
+ let scratch;
1956
+ let path = ownedPath ?? "";
1957
+ let partial = "";
1958
+ let cleanFiles;
1959
+ try {
1960
+ if (options.signal?.aborted)
1961
+ throw cancellationError(options.signal);
1962
+ const { freeBytes, resolveReserveBytes } = await import("./disk.js");
1963
+ const [free, reserve] = await Promise.all([freeBytes(spaceRoot), resolveReserveBytes(dataDir)]);
1964
+ const needed = record.cipher.storedBytes ?? record.remote?.bytes ?? 0;
1965
+ if (free !== undefined && needed > 0 && free < needed + reserve) {
1966
+ const gb = (n) => `${(n / 1024 ** 3).toFixed(1)} GB`;
1967
+ fail("insufficient_disk_space", `Reading archive ${record.id.slice(0, 8)} needs about ${gb(needed)} of temporary space plus ${gb(reserve)} kept free for this machine; only ${gb(free)} is available. Nothing was written. Free space before trying this operation again.`, { archiveId: record.id, free, needed, reserve });
1968
+ }
1969
+ scratch = ownedPath ? undefined : await mkdtemp(join(spaceRoot, "sealkeep-fetch-"));
1970
+ path = ownedPath ?? join(scratch, `${record.id}.skarchive`);
1971
+ partial = `${path}.partial`;
1972
+ cleanFiles = ownedPath
1973
+ ? async () => {
1974
+ await rm(path, { force: true });
1975
+ await rm(partial, { force: true });
1976
+ }
1977
+ : async () => { await rm(scratch, { recursive: true, force: true }); };
1978
+ // A deterministic partial belongs to the same journaled owner as `path`.
1979
+ // A crash may leave it behind; the next attempt removes exactly that file
1980
+ // before writing, and a successful rename never exposes partial bytes.
1981
+ await rm(partial, { force: true });
1982
+ const downloaded = await fetchCiphertextStream(dataDir, record, client, selected, options.overrides, options.signal, undefined, options.maxBytesPerSecond);
1983
+ const cooperative = (async function* () {
1984
+ for await (const chunk of downloaded) {
1985
+ yield chunk;
1986
+ await options.onYield?.();
1987
+ }
1988
+ })();
1989
+ const measured = await writeChunksToFile(partial, cooperative, { mode: 0o600, signal: options.signal, expectedBytes: record.cipher.storedBytes, onProgress: options.onProgress });
1990
+ if (measured.sha256Hex !== record.cipher.ciphertextSha256 || measured.bytes !== record.cipher.storedBytes) {
1991
+ fail("ciphertext_integrity_failed", `The copy of archive ${record.id} in ${selected.bucket} is not the archive that was stored there`, { archiveId: record.id, copyId: selected.copyId });
1992
+ }
1993
+ await rename(partial, path);
1994
+ let releaseComplete = false;
1995
+ let releaseInFlight;
1996
+ return {
1997
+ path,
1998
+ release: () => {
1999
+ if (releaseComplete)
2000
+ return Promise.resolve();
2001
+ if (releaseInFlight)
2002
+ return releaseInFlight;
2003
+ const attempt = (async () => {
2004
+ // Deleting the bytes is the admission boundary. If cleanup fails,
2005
+ // retain both the in-process lane and cross-process lease so another
2006
+ // archive cannot be admitted on top of space that still exists. The
2007
+ // idempotent release handle lets its owner retry the exact cleanup.
2008
+ await cleanFiles();
2009
+ await relinquishPermit();
2010
+ })();
2011
+ releaseInFlight = attempt;
2012
+ attempt.then(() => {
2013
+ releaseComplete = true;
2014
+ if (releaseInFlight === attempt)
2015
+ releaseInFlight = undefined;
2016
+ }, () => { if (releaseInFlight === attempt)
2017
+ releaseInFlight = undefined; });
2018
+ return attempt;
2019
+ },
2020
+ };
2021
+ }
2022
+ catch (error) {
2023
+ let cleanupError;
2024
+ try {
2025
+ if (cleanFiles)
2026
+ await cleanFiles();
2027
+ else {
2028
+ if (partial)
2029
+ await rm(partial, { force: true });
2030
+ if (scratch)
2031
+ await rm(scratch, { recursive: true, force: true });
2032
+ }
2033
+ }
2034
+ catch (cleanupFailure) {
2035
+ cleanupError = cleanupFailure;
2036
+ }
2037
+ if (cleanupError) {
2038
+ // There is no materialised handle to return on a failed fetch, so fail
2039
+ // visibly and keep the reservation for the lifetime of this process.
2040
+ // Releasing it while unknown scratch bytes remain would violate the disk
2041
+ // reserve; a service restart makes the exact-generation lease reclaimable.
2042
+ throw withScratchReleaseFailure(error, cleanupError);
2043
+ }
2044
+ try {
2045
+ await relinquishPermit();
2046
+ }
2047
+ catch (releaseError) {
2048
+ throw withScratchReleaseFailure(error, releaseError);
2049
+ }
2050
+ throw error;
2051
+ }
344
2052
  }