sealkeep 0.8.1 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. package/ARCHITECTURE.md +163 -14
  2. package/CHANGELOG.md +244 -1
  3. package/CONTROL_PLANE.md +2 -2
  4. package/LICENSE +1 -1
  5. package/README.md +108 -22
  6. package/THIRD_PARTY.md +2 -2
  7. package/THREAT_MODEL.md +23 -4
  8. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/chunk-access.d.ts +26 -4
  9. package/dist/packages/sealkeep-crypto/src/chunk-access.js +219 -0
  10. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/envelope.d.ts +11 -1
  11. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/envelope.js +58 -21
  12. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/format.d.ts +2 -2
  13. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/index.d.ts +1 -0
  14. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/index.js +1 -0
  15. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/recipients.js +4 -4
  16. package/dist/packages/sealkeep-crypto/src/squeeze.d.ts +6 -0
  17. package/dist/packages/sealkeep-crypto/src/squeeze.js +39 -0
  18. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/stream.d.ts +36 -3
  19. package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/stream.js +196 -37
  20. package/dist/site/index.html +1808 -1904
  21. package/dist/site/llms.txt +67 -0
  22. package/dist/site/trust/architecture-data-flow.html +53 -0
  23. package/dist/site/trust/audit-roadmap.html +37 -0
  24. package/dist/site/trust/deployment-responsibility.html +11 -0
  25. package/dist/site/trust/dpa-sample.html +30 -0
  26. package/dist/site/trust/release-provenance.html +21 -0
  27. package/dist/site/trust/subprocessors.html +15 -0
  28. package/dist/site/trust/threat-model.html +34 -0
  29. package/dist/site/trust/toms.html +41 -0
  30. package/dist/site/trust-document.css +32 -0
  31. package/dist/site/trust.html +73 -0
  32. package/dist/site/visual/assets/index-8Nxnkc7o.js +207 -0
  33. package/dist/site/visual/assets/index-oxLy1bpy.css +1 -0
  34. package/dist/site/visual/index.html +18 -0
  35. package/dist/site.zip +0 -0
  36. package/dist/src/activity.d.ts +9 -0
  37. package/dist/src/activity.js +90 -1
  38. package/dist/src/adapters.d.ts +175 -5
  39. package/dist/src/adapters.js +961 -73
  40. package/dist/src/agent-context.d.ts +135 -0
  41. package/dist/src/agent-context.js +1059 -0
  42. package/dist/src/archive-copies.d.ts +47 -0
  43. package/dist/src/archive-copies.js +179 -0
  44. package/dist/src/audit.d.ts +1 -1
  45. package/dist/src/audit.js +29 -4
  46. package/dist/src/autopilot.d.ts +52 -7
  47. package/dist/src/autopilot.js +143 -25
  48. package/dist/src/background-bandwidth.d.ts +46 -0
  49. package/dist/src/background-bandwidth.js +301 -0
  50. package/dist/src/background-cpu.d.ts +82 -0
  51. package/dist/src/background-cpu.js +212 -0
  52. package/dist/src/background-worker-error.d.ts +12 -0
  53. package/dist/src/background-worker-error.js +18 -0
  54. package/dist/src/branding.d.ts +24 -1
  55. package/dist/src/branding.js +31 -1
  56. package/dist/src/bridge.d.ts +233 -0
  57. package/dist/src/bridge.js +604 -0
  58. package/dist/src/byte-stream.d.ts +91 -0
  59. package/dist/src/byte-stream.js +385 -0
  60. package/dist/src/chunk-store.d.ts +41 -8
  61. package/dist/src/chunk-store.js +161 -65
  62. package/dist/src/cli.js +1746 -166
  63. package/dist/src/cloud.d.ts +841 -31
  64. package/dist/src/cloud.js +3196 -277
  65. package/dist/src/context-background.d.ts +37 -0
  66. package/dist/src/context-background.js +309 -0
  67. package/dist/src/context-drain-child.d.ts +1 -0
  68. package/dist/src/context-drain-child.js +98 -0
  69. package/dist/src/context-reader.d.ts +118 -0
  70. package/dist/src/context-reader.js +447 -0
  71. package/dist/src/control-plane/auth.d.ts +32 -4
  72. package/dist/src/control-plane/auth.js +85 -24
  73. package/dist/src/control-plane/server.js +19 -6
  74. package/dist/src/control-plane.d.ts +17 -1
  75. package/dist/src/control-plane.js +32 -6
  76. package/dist/src/crypto.d.ts +1 -1
  77. package/dist/src/crypto.js +5 -5
  78. package/dist/src/daemon-lease.d.ts +70 -0
  79. package/dist/src/daemon-lease.js +420 -0
  80. package/dist/src/daemon.d.ts +94 -1
  81. package/dist/src/daemon.js +1082 -105
  82. package/dist/src/darwin-service-policy.d.ts +41 -0
  83. package/dist/src/darwin-service-policy.js +60 -0
  84. package/dist/src/dashboard-cli.js +15 -15
  85. package/dist/src/device-authorization.d.ts +37 -0
  86. package/dist/src/device-authorization.js +199 -0
  87. package/dist/src/device-enrollment.d.ts +91 -0
  88. package/dist/src/device-enrollment.js +349 -0
  89. package/dist/src/disk.d.ts +17 -12
  90. package/dist/src/disk.js +43 -17
  91. package/dist/src/doctor.d.ts +35 -1
  92. package/dist/src/doctor.js +316 -41
  93. package/dist/src/durable-ticket-lock.d.ts +24 -0
  94. package/dist/src/durable-ticket-lock.js +232 -0
  95. package/dist/src/enroll.d.ts +1 -1
  96. package/dist/src/enroll.js +13 -7
  97. package/dist/src/env.d.ts +10 -1
  98. package/dist/src/env.js +11 -3
  99. package/dist/src/errors.d.ts +8 -8
  100. package/dist/src/errors.js +6 -6
  101. package/dist/src/flush.d.ts +12 -0
  102. package/dist/src/flush.js +37 -0
  103. package/dist/src/heartbeat.d.ts +86 -12
  104. package/dist/src/heartbeat.js +415 -29
  105. package/dist/src/index-background-watchdog.d.ts +1 -0
  106. package/dist/src/index-background-watchdog.js +94 -0
  107. package/dist/src/index-background-work.d.ts +21 -0
  108. package/dist/src/index-background-work.js +25 -0
  109. package/dist/src/index-background.d.ts +64 -0
  110. package/dist/src/index-background.js +394 -0
  111. package/dist/src/index-build-child.d.ts +1 -0
  112. package/dist/src/index-build-child.js +109 -0
  113. package/dist/src/index-manifest.d.ts +52 -0
  114. package/dist/src/index-manifest.js +444 -0
  115. package/dist/src/index-publication-proof.d.ts +84 -0
  116. package/dist/src/index-publication-proof.js +380 -0
  117. package/dist/src/index-publication-state.d.ts +149 -0
  118. package/dist/src/index-publication-state.js +696 -0
  119. package/dist/src/index-publication-verifier.d.ts +89 -0
  120. package/dist/src/index-publication-verifier.js +341 -0
  121. package/dist/src/index-publish.d.ts +62 -0
  122. package/dist/src/index-publish.js +540 -0
  123. package/dist/src/index-scratch-cleanup.d.ts +19 -0
  124. package/dist/src/index-scratch-cleanup.js +166 -0
  125. package/dist/src/index-segment-types.d.ts +132 -0
  126. package/dist/src/index-segment-types.js +21 -0
  127. package/dist/src/index-segments.d.ts +9 -0
  128. package/dist/src/index-segments.js +516 -0
  129. package/dist/src/index-store.d.ts +123 -0
  130. package/dist/src/index-store.js +495 -0
  131. package/dist/src/index-sync.d.ts +91 -15
  132. package/dist/src/index-sync.js +286 -53
  133. package/dist/src/index-upgrade-publication.d.ts +30 -0
  134. package/dist/src/index-upgrade-publication.js +179 -0
  135. package/dist/src/integration-manager.d.ts +32 -0
  136. package/dist/src/integration-manager.js +394 -0
  137. package/dist/src/leakscan.js +1 -1
  138. package/dist/src/local-api.d.ts +56 -15
  139. package/dist/src/local-api.js +4979 -541
  140. package/dist/src/machine-settings.d.ts +51 -0
  141. package/dist/src/machine-settings.js +166 -0
  142. package/dist/src/managed-chunks.d.ts +5 -2
  143. package/dist/src/managed-chunks.js +14 -14
  144. package/dist/src/mcp-install.d.ts +11 -9
  145. package/dist/src/mcp-install.js +73 -28
  146. package/dist/src/mcp-workspace.d.ts +18 -0
  147. package/dist/src/mcp-workspace.js +50 -0
  148. package/dist/src/mcp.js +294 -25
  149. package/dist/src/migrate.d.ts +1 -0
  150. package/dist/src/migrate.js +33 -5
  151. package/dist/src/notify.d.ts +1 -1
  152. package/dist/src/notify.js +5 -5
  153. package/dist/src/offload.d.ts +201 -14
  154. package/dist/src/offload.js +1848 -140
  155. package/dist/src/onboarding.d.ts +8 -1
  156. package/dist/src/onboarding.js +4 -4
  157. package/dist/src/packages.d.ts +2 -2
  158. package/dist/src/packages.js +10 -2
  159. package/dist/src/passkey.d.ts +0 -1
  160. package/dist/src/passkey.js +2 -7
  161. package/dist/src/password-lock.d.ts +2 -2
  162. package/dist/src/password-lock.js +6 -6
  163. package/dist/src/paths.d.ts +2 -0
  164. package/dist/src/paths.js +2 -0
  165. package/dist/src/presence.d.ts +86 -0
  166. package/dist/src/presence.js +240 -0
  167. package/dist/src/progress-deadline.d.ts +21 -0
  168. package/dist/src/progress-deadline.js +91 -0
  169. package/dist/src/project-repair.d.ts +55 -0
  170. package/dist/src/project-repair.js +131 -0
  171. package/dist/src/providers/gcs.d.ts +28 -7
  172. package/dist/src/providers/gcs.js +35 -24
  173. package/dist/src/providers/gdrive.d.ts +71 -8
  174. package/dist/src/providers/gdrive.js +223 -50
  175. package/dist/src/providers/index.d.ts +11 -3
  176. package/dist/src/providers/index.js +30 -10
  177. package/dist/src/providers/s3.d.ts +30 -8
  178. package/dist/src/providers/s3.js +41 -30
  179. package/dist/src/providers/safe-storage-fetch.d.ts +12 -0
  180. package/dist/src/providers/safe-storage-fetch.js +72 -0
  181. package/dist/src/queue.d.ts +136 -19
  182. package/dist/src/queue.js +862 -96
  183. package/dist/src/reclaim-transaction.d.ts +156 -0
  184. package/dist/src/reclaim-transaction.js +1027 -0
  185. package/dist/src/recovery-codes.d.ts +32 -0
  186. package/dist/src/recovery-codes.js +338 -0
  187. package/dist/src/recovery.js +12 -9
  188. package/dist/src/rehydrate.d.ts +25 -22
  189. package/dist/src/rehydrate.js +319 -23
  190. package/dist/src/restore.d.ts +57 -4
  191. package/dist/src/restore.js +272 -36
  192. package/dist/src/resume-stub.d.ts +92 -0
  193. package/dist/src/resume-stub.js +417 -0
  194. package/dist/src/retention.d.ts +98 -7
  195. package/dist/src/retention.js +1066 -62
  196. package/dist/src/rotate.js +3 -3
  197. package/dist/src/search.d.ts +566 -8
  198. package/dist/src/search.js +5897 -290
  199. package/dist/src/secrets.d.ts +51 -7
  200. package/dist/src/secrets.js +316 -24
  201. package/dist/src/service.d.ts +49 -11
  202. package/dist/src/service.js +776 -35
  203. package/dist/src/share.js +3 -3
  204. package/dist/src/shared-spaces.d.ts +98 -0
  205. package/dist/src/shared-spaces.js +214 -0
  206. package/dist/src/source-reader.d.ts +73 -0
  207. package/dist/src/source-reader.js +715 -0
  208. package/dist/src/spool.d.ts +1 -1
  209. package/dist/src/spool.js +1 -1
  210. package/dist/src/start-tui.js +2 -1
  211. package/dist/src/start.js +2 -2
  212. package/dist/src/storage-endpoint.d.ts +21 -0
  213. package/dist/src/storage-endpoint.js +122 -0
  214. package/dist/src/storage-setup.js +12 -12
  215. package/dist/src/storage-targets.d.ts +109 -6
  216. package/dist/src/storage-targets.js +975 -67
  217. package/dist/src/stream-to-cloud.d.ts +5 -1
  218. package/dist/src/stream-to-cloud.js +34 -14
  219. package/dist/src/sync-rules.d.ts +31 -6
  220. package/dist/src/sync-rules.js +153 -14
  221. package/dist/src/team-backfill-scheduling.d.ts +8 -0
  222. package/dist/src/team-backfill-scheduling.js +33 -0
  223. package/dist/src/team-backfill.d.ts +116 -0
  224. package/dist/src/team-backfill.js +1429 -0
  225. package/dist/src/team-index-cache.d.ts +16 -0
  226. package/dist/src/team-index-cache.js +152 -0
  227. package/dist/src/team-offboarding.d.ts +38 -0
  228. package/dist/src/team-offboarding.js +1043 -0
  229. package/dist/src/team-presence.d.ts +127 -0
  230. package/dist/src/team-presence.js +904 -0
  231. package/dist/src/team-publication-policy.d.ts +20 -0
  232. package/dist/src/team-publication-policy.js +140 -0
  233. package/dist/src/team-realtime.d.ts +68 -0
  234. package/dist/src/team-realtime.js +816 -0
  235. package/dist/src/team-source-facts-cache.d.ts +23 -0
  236. package/dist/src/team-source-facts-cache.js +255 -0
  237. package/dist/src/trash.d.ts +1 -1
  238. package/dist/src/trash.js +2 -2
  239. package/dist/src/tui.js +11 -12
  240. package/dist/src/types.d.ts +173 -7
  241. package/dist/src/types.js +20 -0
  242. package/dist/src/ui-server.d.ts +163 -35
  243. package/dist/src/ui-server.js +712 -72
  244. package/dist/src/ui.d.ts +1 -2
  245. package/dist/src/ui.js +1 -2
  246. package/dist/src/upload.d.ts +27 -0
  247. package/dist/src/upload.js +383 -43
  248. package/dist/src/vault.d.ts +235 -29
  249. package/dist/src/vault.js +1783 -190
  250. package/dist/src/watcher.d.ts +7 -1
  251. package/dist/src/watcher.js +198 -55
  252. package/dist/src/worker.d.ts +27 -3
  253. package/dist/src/worker.js +274 -55
  254. package/package.json +33 -12
  255. package/scripts/native-reboot-rehearsal.mjs +90 -0
  256. package/web/app.js +6032 -343
  257. package/web/bootstrap.js +17 -0
  258. package/web/index.html +255 -57
  259. package/web/rail.js +317 -40
  260. package/web/retention.html +2 -2
  261. package/web/rules-view.js +188 -16
  262. package/web/sessions-view.js +485 -62
  263. package/web/sessions.html +2 -2
  264. package/web/setup-api.js +152 -29
  265. package/web/setup-logic.js +68 -9
  266. package/web/setup.html +113 -44
  267. package/web/setup.js +604 -71
  268. package/web/style.css +513 -98
  269. package/dist/packages/vaultline-crypto/src/chunk-access.js +0 -93
  270. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/aead.d.ts +0 -0
  271. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/aead.js +0 -0
  272. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/format.js +0 -0
  273. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/recipients.d.ts +0 -0
  274. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/sha256-stream.d.ts +0 -0
  275. /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/sha256-stream.js +0 -0
@@ -1,8 +1,39 @@
1
- import { readFile, rename, writeFile } from "node:fs/promises";
1
+ import { readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
2
2
  import { randomUUID } from "node:crypto";
3
3
  import { join } from "node:path";
4
+ import { isV2 } from "./types.js";
5
+ import { archiveCopies, hasCompleteVerifiedCopyChain } from "./archive-copies.js";
6
+ import { readConfig, resolveDeltaChain } from "./vault.js";
7
+ import { backgroundUploadBytesPerSecond, readLocalSettings } from "./machine-settings.js";
4
8
  export const heartbeatPath = (dataDir) => join(dataDir, "daemon-state.json");
9
+ const STALE_TEMPORARY_MS = 60_000;
10
+ const sweptRoots = new Set();
11
+ /**
12
+ * Every JSON file in the vault root is replaced through `<name>.<uuid>.tmp`
13
+ * and rename. A daemon stopped between the two — a service reload, a reboot —
14
+ * leaves the temporary behind, and nothing ever looked for it again: a vault
15
+ * carried one per restart, forever. The first heartbeat of a process removes
16
+ * the ones no live writer can still own; a minute is far longer than any of
17
+ * these writes takes, and the sweep is scoped to that naming so it can never
18
+ * touch a real file.
19
+ */
20
+ async function sweepStaleTemporaries(dataDir) {
21
+ if (sweptRoots.has(dataDir))
22
+ return;
23
+ sweptRoots.add(dataDir);
24
+ const names = await readdir(dataDir).catch(() => []);
25
+ const now = Date.now();
26
+ for (const name of names) {
27
+ if (!/\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.tmp$/.test(name))
28
+ continue;
29
+ const path = join(dataDir, name);
30
+ const modifiedAt = await stat(path).then((value) => value.mtimeMs).catch(() => Number.NaN);
31
+ if (Number.isFinite(modifiedAt) && now - modifiedAt > STALE_TEMPORARY_MS)
32
+ await rm(path, { force: true }).catch(() => undefined);
33
+ }
34
+ }
5
35
  export async function writeHeartbeat(dataDir, beat) {
36
+ await sweepStaleTemporaries(dataDir);
6
37
  const path = heartbeatPath(dataDir);
7
38
  const temp = `${path}.${randomUUID()}.tmp`;
8
39
  await writeFile(temp, JSON.stringify(beat, null, 2) + "\n", { mode: 0o600 });
@@ -49,63 +80,418 @@ export function liveness(beat, now = Date.now(), alive = processAlive) {
49
80
  }
50
81
  return {
51
82
  state: "running",
52
- detail: beat.paused ? "Running but paused; no work is being done." : `Running. Last checked ${describeGap(secondsSinceTick)}.`,
83
+ detail: beat.paused
84
+ ? "Running. New work is paused; an operation already in progress may finish safely."
85
+ : `Running. Last checked ${describeGap(secondsSinceTick)}.${beat.teamAccess?.state === "attention" ? " Team access needs attention; retrying automatically." : ""}`,
53
86
  lastTickAt: beat.lastTickAt, secondsSinceTick, paused: beat.paused, totals: beat.totals
54
87
  };
55
88
  }
89
+ const uploadRetryPath = (dataDir) => join(dataDir, "upload-retries.json");
90
+ const archiveCacheRetryPath = (dataDir) => join(dataDir, "archive-cache-retries.json");
91
+ async function readUploadRetries(dataDir) {
92
+ try {
93
+ const parsed = JSON.parse(await readFile(uploadRetryPath(dataDir), "utf8"));
94
+ return parsed.version === 1 && parsed.archives && typeof parsed.archives === "object"
95
+ ? { version: 1, archives: parsed.archives }
96
+ : { version: 1, archives: {} };
97
+ }
98
+ catch {
99
+ return { version: 1, archives: {} };
100
+ }
101
+ }
102
+ async function writeUploadRetries(dataDir, ledger) {
103
+ const path = uploadRetryPath(dataDir);
104
+ const temp = `${path}.${randomUUID()}.tmp`;
105
+ await writeFile(temp, JSON.stringify(ledger, null, 2) + "\n", { mode: 0o600 });
106
+ await rename(temp, path);
107
+ }
108
+ async function readArchiveCacheRetries(dataDir) {
109
+ try {
110
+ const parsed = JSON.parse(await readFile(archiveCacheRetryPath(dataDir), "utf8"));
111
+ return parsed.version === 1 && parsed.archives && typeof parsed.archives === "object"
112
+ ? { version: 1, archives: parsed.archives }
113
+ : { version: 1, archives: {} };
114
+ }
115
+ catch {
116
+ return { version: 1, archives: {} };
117
+ }
118
+ }
119
+ async function writeArchiveCacheRetries(dataDir, ledger) {
120
+ const path = archiveCacheRetryPath(dataDir);
121
+ const temp = `${path}.${randomUUID()}.tmp`;
122
+ await writeFile(temp, JSON.stringify(ledger, null, 2) + "\n", { mode: 0o600 });
123
+ await rename(temp, path);
124
+ }
125
+ function nextUploadAttempt(attempts, now) {
126
+ // Five seconds makes one-off network failures cheap; exponential cooling
127
+ // stops a permanent first page of failures from consuming every bounded pass.
128
+ const delay = Math.min(60 * 60_000, 5_000 * 2 ** Math.min(Math.max(0, attempts - 1), 10));
129
+ return new Date(now + delay).toISOString();
130
+ }
131
+ /**
132
+ * Gives the daemon a bounded, retry-cooled cleanup lane for encrypted local
133
+ * duplicates. This runs even when the upload backlog is empty, so a service
134
+ * upgrade or an earlier provider outage cannot strand old `.skarchive` files.
135
+ */
136
+ export async function sweepExpiredArchiveCache(dataDir, options) {
137
+ const offloadModule = await import("./offload.js");
138
+ // Transaction recovery is independent of whether this machine currently
139
+ // retains new encrypted archives forever. A crash after unlinking the
140
+ // source must converge on the next daemon pass even if the cache policy was
141
+ // changed meanwhile. Injected test offload functions remain self-contained.
142
+ const recovered = options.offload
143
+ ? { completed: [], freedBytes: 0 }
144
+ : await offloadModule.reconcileOffloadTransactions(dataDir, {
145
+ client: options.client,
146
+ maxBytesPerSecond: options.maxBytesPerSecond,
147
+ signal: options.signal,
148
+ });
149
+ const recoveredIds = recovered.completed.map((candidate) => candidate.id);
150
+ if (options.cacheDays === null)
151
+ return { offloaded: recoveredIds, freedBytes: recovered.freedBytes };
152
+ const now = options.now ?? Date.now();
153
+ const limit = Math.max(0, Math.trunc(options.maxArchives ?? 8));
154
+ if (limit === 0)
155
+ return { offloaded: recoveredIds, freedBytes: recovered.freedBytes };
156
+ const { offloadable, offloadArchives: realOffload } = offloadModule;
157
+ const offload = options.offload ?? realOffload;
158
+ const preview = await offloadable(dataDir, { olderThanDays: options.cacheDays, now });
159
+ const candidateIds = new Set(preview.candidates.map((candidate) => candidate.id));
160
+ const retries = await readArchiveCacheRetries(dataDir);
161
+ let retriesChanged = false;
162
+ for (const archiveId of Object.keys(retries.archives)) {
163
+ if (!candidateIds.has(archiveId)) {
164
+ delete retries.archives[archiveId];
165
+ retriesChanged = true;
166
+ }
167
+ }
168
+ // listArchives is newest-first; reversing gives the oldest expired cache
169
+ // entries first, which makes bounded passes recover space fairly.
170
+ const selected = [...preview.candidates].reverse().filter((candidate) => {
171
+ const retry = retries.archives[candidate.id];
172
+ return !retry || Date.parse(retry.nextAttemptAt) <= now;
173
+ }).slice(0, limit);
174
+ if (selected.length === 0) {
175
+ if (retriesChanged)
176
+ await writeArchiveCacheRetries(dataDir, retries).catch(() => undefined);
177
+ return { offloaded: recoveredIds, freedBytes: recovered.freedBytes };
178
+ }
179
+ const selectedIds = new Set(selected.map((candidate) => candidate.id));
180
+ const markRetry = (archiveId, error) => {
181
+ const attempts = (retries.archives[archiveId]?.attempts ?? 0) + 1;
182
+ retries.archives[archiveId] = {
183
+ attempts,
184
+ nextAttemptAt: nextUploadAttempt(attempts, now),
185
+ lastError: error.replace(/\s+/g, " ").slice(0, 500),
186
+ };
187
+ retriesChanged = true;
188
+ };
189
+ try {
190
+ options.signal?.throwIfAborted();
191
+ const outcome = await offload(dataDir, {
192
+ confirm: true,
193
+ olderThanDays: options.cacheDays,
194
+ now,
195
+ only: selectedIds,
196
+ freshlyVerified: options.freshlyVerified,
197
+ signal: options.signal,
198
+ client: options.client,
199
+ maxBytesPerSecond: options.maxBytesPerSecond,
200
+ });
201
+ if (outcome.mode !== "apply")
202
+ throw new Error("archive cache cleanup did not enter apply mode");
203
+ const completed = new Set(outcome.offloaded.map((candidate) => candidate.id));
204
+ const failures = new Map([
205
+ ...outcome.failed.map((failure) => [failure.id, failure.error]),
206
+ ...outcome.blocked.map((failure) => [failure.id, failure.reason]),
207
+ ]);
208
+ for (const archiveId of selectedIds) {
209
+ if (completed.has(archiveId)) {
210
+ if (retries.archives[archiveId]) {
211
+ delete retries.archives[archiveId];
212
+ retriesChanged = true;
213
+ }
214
+ }
215
+ else {
216
+ markRetry(archiveId, failures.get(archiveId) ?? "verified archive cache cleanup did not complete");
217
+ }
218
+ }
219
+ if (retriesChanged)
220
+ await writeArchiveCacheRetries(dataDir, retries).catch(() => undefined);
221
+ return { offloaded: [...new Set([...recoveredIds, ...completed])], freedBytes: recovered.freedBytes + outcome.freedBytes };
222
+ }
223
+ catch (error) {
224
+ if (options.signal?.aborted)
225
+ throw options.signal.reason ?? error;
226
+ const message = error instanceof Error ? error.message : "archive cache cleanup failed";
227
+ for (const archiveId of selectedIds)
228
+ markRetry(archiveId, message);
229
+ await writeArchiveCacheRetries(dataDir, retries).catch(() => undefined);
230
+ return { offloaded: recoveredIds, freedBytes: recovered.freedBytes };
231
+ }
232
+ }
233
+ function uploadChains(records) {
234
+ const byChain = new Map();
235
+ for (const record of records) {
236
+ if (!isV2(record))
237
+ continue;
238
+ let chain;
239
+ try {
240
+ chain = resolveDeltaChain(records, record);
241
+ }
242
+ catch {
243
+ continue;
244
+ }
245
+ if (chain.length < 2)
246
+ continue;
247
+ const rootId = chain[0].id;
248
+ if (!byChain.has(rootId))
249
+ byChain.set(rootId, { rootId, records: chain });
250
+ }
251
+ const chains = [...byChain.values()];
252
+ const result = new Map();
253
+ for (const chain of chains)
254
+ for (const record of chain.records)
255
+ result.set(record.id, chain);
256
+ return { byId: result, chains };
257
+ }
258
+ async function chooseCommonTarget(dataDir, chain, config, options = {}) {
259
+ const { chooseTarget, remoteCopyMatchesTarget, resolveTargets, targetUsage } = await import("./storage-targets.js");
260
+ const targets = await resolveTargets(dataDir);
261
+ for (const target of targets) {
262
+ let additional = 0;
263
+ let eligible = true;
264
+ for (const item of chain.records) {
265
+ if (!isV2(item)) {
266
+ eligible = false;
267
+ break;
268
+ }
269
+ const teamBound = Boolean(item.source.projectScope || (item.source.project && config.teamSpaces?.[item.source.project]));
270
+ let alreadyThere = false;
271
+ for (const copy of archiveCopies(item)) {
272
+ if (copy.verifiedAt && await remoteCopyMatchesTarget(dataDir, target, copy)) {
273
+ alreadyThere = true;
274
+ break;
275
+ }
276
+ }
277
+ if (alreadyThere)
278
+ continue;
279
+ // Injected clients are the provider-test seam and already prove local
280
+ // connectivity. Keep every routing/cap/team check, but do not require a
281
+ // machine credential that the seam intentionally replaces.
282
+ if (options.clientProvided) {
283
+ if (target.enabled === false || (teamBound && target.provider !== "vaultline")) {
284
+ eligible = false;
285
+ break;
286
+ }
287
+ if (target.maxGb !== undefined) {
288
+ const usage = (await targetUsage(dataDir, targets)).get(target.id) ?? 0;
289
+ if (usage + additional + item.cipher.storedBytes > target.maxGb * 1024 ** 3) {
290
+ eligible = false;
291
+ break;
292
+ }
293
+ }
294
+ }
295
+ else
296
+ try {
297
+ // Include earlier missing links in the projected size so maxGb is
298
+ // checked for the whole chain, not once per link against stale usage.
299
+ await chooseTarget(dataDir, {
300
+ project: item.source.project ?? null,
301
+ projectKey: item.source.projectKey ?? null,
302
+ bytes: item.cipher.storedBytes + additional,
303
+ requireManaged: teamBound,
304
+ targetId: target.id,
305
+ });
306
+ }
307
+ catch {
308
+ eligible = false;
309
+ break;
310
+ }
311
+ additional += item.cipher.storedBytes;
312
+ }
313
+ if (eligible)
314
+ return target.id;
315
+ }
316
+ return null;
317
+ }
56
318
  /**
57
319
  * The daemon's upload pass, finished properly: upload, verify, then offload.
58
320
  *
59
321
  * `uploadPending` alone leaves a sealed local copy of every archive it just
60
- * proved the bucket holds — on the machines this product is for, that is the
61
- * disk staying exactly as full as before. The dashboard's push already does
62
- * the whole job (see `runUploads` in local-api.ts): the moment an archive's
63
- * verify lands, its local blob is a duplicate, and offloading it right then
64
- * lets a full disk breathe with each verify instead of never.
322
+ * proved the bucket holds — on the machines this product is for, that cache
323
+ * must not grow forever. The visible per-machine cache setting decides when a
324
+ * verified duplicate leaves; a value of zero completes that cleanup in this
325
+ * same pass.
65
326
  *
66
327
  * Offload failure is tolerated per archive: the upload is still verified and
67
- * recorded, and the next offload sweep gets another chance. `freshlyVerified`
68
- * is what keeps this cheap — the verify happened milliseconds ago, so the
69
- * usual fetch-and-hash re-proof would double the egress of every push.
328
+ * recorded, and a bounded, backed-off sweep gets another chance. A fresh
329
+ * verify still receives one final streamed whole-chain proof at the actual
330
+ * local purge boundary when the cache setting is zero.
70
331
  *
71
- * Integration note: src/daemon.ts's tick still calls bare `uploadPending`;
72
- * switching it to this function is the remaining wiring (daemon.ts is outside
73
- * this change's file list).
332
+ * The daemon calls this combined pass so routing, remote verification, retry
333
+ * fairness, and reclaim all share one ordering boundary.
74
334
  */
75
335
  export async function uploadAndOffloadPending(dataDir, options = {}) {
76
336
  // Imported lazily, like the paths local-api takes: this module is read by
77
337
  // every surface that asks "is the service alive", none of which should pay
78
338
  // for the provider stack.
79
- const [{ uploadArchive }, { listArchives }, { isV2 }] = await Promise.all([
80
- import("./upload.js"), import("./vault.js"), import("./types.js")
339
+ const [{ uploadArchive }, { listArchives }, { hasVerifiedCopy: hasVerifiedCopyDynamic }] = await Promise.all([
340
+ import("./upload.js"), import("./vault.js"), import("./archive-copies.js")
81
341
  ]);
82
- const { offload: injectedOffload, ...uploadOptions } = options;
342
+ const { offload: injectedOffload, maxArchives, maxOffloads, ...uploadOptions } = options;
83
343
  const offload = injectedOffload ?? (await import("./offload.js")).offloadArchives;
84
- const records = await listArchives(dataDir);
344
+ const localSettings = await readLocalSettings(dataDir);
345
+ // This function is the daemon's automatic upload boundary. A null field in
346
+ // older settings meant "reserved/not implemented", not informed consent to
347
+ // saturate the link, so it resolves to the safe automatic rate here.
348
+ uploadOptions.maxBytesPerSecond ??= backgroundUploadBytesPerSecond(localSettings);
349
+ // listArchives is newest-first for the UI. Work oldest-first here so a large
350
+ // new session (or a failure on it) cannot keep earlier memory local forever.
351
+ const records = (await listArchives(dataDir))
352
+ .sort((a, b) => a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id));
353
+ const config = await readConfig(dataDir);
354
+ const chainIndex = uploadChains(records);
355
+ const incompleteChainById = new Map();
356
+ for (const chain of chainIndex.chains) {
357
+ if (chain.records.every(isV2) && hasCompleteVerifiedCopyChain(chain.records))
358
+ continue;
359
+ for (const item of chain.records)
360
+ incompleteChainById.set(item.id, chain);
361
+ }
362
+ const retries = await readUploadRetries(dataDir);
363
+ const pendingIds = new Set(records.filter((record) => !isV2(record) || incompleteChainById.has(record.id) || !hasVerifiedCopyDynamic(record)).map((record) => record.id));
364
+ let retriesChanged = false;
365
+ for (const archiveId of Object.keys(retries.archives)) {
366
+ if (!pendingIds.has(archiveId)) {
367
+ delete retries.archives[archiveId];
368
+ retriesChanged = true;
369
+ }
370
+ }
85
371
  const uploaded = [];
86
372
  const failed = [];
87
373
  const offloaded = [];
88
374
  let freedBytes = 0;
375
+ let attempted = 0;
376
+ const commonTargets = new Map();
377
+ const blockedChains = new Set();
378
+ const limit = maxArchives === undefined ? Number.POSITIVE_INFINITY : Math.max(0, Math.trunc(maxArchives));
89
379
  for (const record of records) {
90
- if (isV2(record) && record.remote?.verifiedAt)
380
+ uploadOptions.signal?.throwIfAborted();
381
+ const chain = incompleteChainById.get(record.id);
382
+ if (isV2(record) && hasVerifiedCopyDynamic(record) && !chain)
91
383
  continue;
92
- try {
93
- uploaded.push(await uploadArchive(dataDir, record.id, uploadOptions));
94
- }
95
- catch (error) {
96
- failed.push({ archiveId: record.id, error: error instanceof Error ? error.message : "upload failed" });
384
+ if (chain && blockedChains.has(chain.rootId))
97
385
  continue;
386
+ const now = Date.now();
387
+ const retry = retries.archives[record.id];
388
+ if (retry && Date.parse(retry.nextAttemptAt) > now)
389
+ continue;
390
+ if (attempted >= limit)
391
+ break;
392
+ let targetId;
393
+ if (chain) {
394
+ if (!commonTargets.has(chain.rootId)) {
395
+ commonTargets.set(chain.rootId, await chooseCommonTarget(dataDir, chain, config, { clientProvided: Boolean(uploadOptions.client) }));
396
+ }
397
+ const chosen = commonTargets.get(chain.rootId);
398
+ if (!chosen) {
399
+ blockedChains.add(chain.rootId);
400
+ const message = "No one eligible storage destination can hold the complete archive chain yet; the chain remains local until a common destination is available.";
401
+ failed.push({ archiveId: record.id, error: message });
402
+ const attempts = (retry?.attempts ?? 0) + 1;
403
+ retries.archives[record.id] = { attempts, nextAttemptAt: nextUploadAttempt(attempts, now), lastError: message };
404
+ retriesChanged = true;
405
+ continue;
406
+ }
407
+ targetId = chosen;
408
+ // A verified copy at the frozen destination needs no transfer. It still
409
+ // belongs to the chain's work set, so leave it available for the common
410
+ // destination proof but do not upload it again.
411
+ let alreadyThere = false;
412
+ const { resolveTargets, remoteCopyMatchesTarget } = await import("./storage-targets.js");
413
+ const target = (await resolveTargets(dataDir)).find((candidate) => candidate.id === targetId);
414
+ if (target)
415
+ for (const copy of isV2(record) ? archiveCopies(record) : []) {
416
+ if (copy.verifiedAt && await remoteCopyMatchesTarget(dataDir, target, copy)) {
417
+ alreadyThere = true;
418
+ break;
419
+ }
420
+ }
421
+ if (alreadyThere)
422
+ continue;
98
423
  }
424
+ attempted += 1;
99
425
  try {
100
- const outcome = await offload(dataDir, { confirm: true, only: new Set([record.id]), freshlyVerified: new Set([record.id]) });
101
- if (outcome.mode === "apply") {
102
- freedBytes += outcome.freedBytes;
103
- offloaded.push(...outcome.offloaded.map((item) => item.id));
426
+ // An injected provider client is a test seam, not a machine credential
427
+ // for the named target. Keep the frozen target for eligibility above,
428
+ // while letting the seam use its explicit transport without requiring a
429
+ // persisted credential binding. Production calls always carry targetId.
430
+ const outcome = await uploadArchive(dataDir, record.id, {
431
+ ...uploadOptions,
432
+ ...(targetId && !uploadOptions.client ? { targetId } : {}),
433
+ });
434
+ if (!outcome.verified)
435
+ throw new Error("storage did not verify this upload as durable");
436
+ uploaded.push(outcome);
437
+ if (retry) {
438
+ delete retries.archives[record.id];
439
+ retriesChanged = true;
104
440
  }
105
441
  }
106
- catch { /* offload is a bonus on this pass; the sweep gets another chance */ }
442
+ catch (error) {
443
+ if (uploadOptions.signal?.aborted)
444
+ throw uploadOptions.signal.reason ?? error;
445
+ const message = error instanceof Error ? error.message : "upload failed";
446
+ failed.push({ archiveId: record.id, error: message });
447
+ const attempts = (retry?.attempts ?? 0) + 1;
448
+ retries.archives[record.id] = {
449
+ attempts,
450
+ nextAttemptAt: nextUploadAttempt(attempts, now),
451
+ lastError: message.replace(/\s+/g, " ").slice(0, 500),
452
+ };
453
+ retriesChanged = true;
454
+ continue;
455
+ }
456
+ }
457
+ if (retriesChanged)
458
+ await writeUploadRetries(dataDir, retries).catch(() => undefined);
459
+ // The per-machine cache policy is visible in Settings. It is deliberately
460
+ // separate from source retention: this removes only encrypted `.skarchive`
461
+ // duplicates that a readable off-machine destination has already verified.
462
+ const cacheSweep = await sweepExpiredArchiveCache(dataDir, {
463
+ cacheDays: localSettings.localArchiveCacheDays,
464
+ offload,
465
+ maxArchives: maxOffloads ?? Math.min(8, limit),
466
+ freshlyVerified: new Set(uploaded.map((item) => item.archiveId)),
467
+ signal: uploadOptions.signal,
468
+ client: uploadOptions.client,
469
+ maxBytesPerSecond: uploadOptions.maxBytesPerSecond,
470
+ });
471
+ offloaded.push(...cacheSweep.offloaded);
472
+ freedBytes += cacheSweep.freedBytes;
473
+ // A team grant is separate from account durability. Retry it every daemon
474
+ // pass so an upload completed while offline, a crash after verification, or
475
+ // a newly enabled project all converge without a Sealkeep command.
476
+ let reconciled;
477
+ try {
478
+ reconciled = await (await import("./upload.js")).reconcileManagedTeamArchives(dataDir, {
479
+ signal: uploadOptions.signal,
480
+ });
107
481
  }
108
- return { uploaded, failed, skipped: records.length - uploaded.length - failed.length, offloaded, freedBytes };
482
+ catch (error) {
483
+ if (uploadOptions.signal?.aborted)
484
+ throw uploadOptions.signal.reason ?? error;
485
+ reconciled = { registered: 0, failed: 0 };
486
+ }
487
+ return {
488
+ uploaded,
489
+ failed,
490
+ skipped: records.length - uploaded.length - failed.length,
491
+ offloaded,
492
+ freedBytes,
493
+ teamRegistered: reconciled.registered,
494
+ };
109
495
  }
110
496
  export function describeGap(seconds) {
111
497
  if (seconds < 5)
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,94 @@
1
+ import { daemonProcessIdentity } from "./daemon-lease.js";
2
+ let ownership;
3
+ let disconnected = false;
4
+ let disarmed = false;
5
+ let cleanupStarted = false;
6
+ let arming;
7
+ async function signalOwned(signal) {
8
+ if (!ownership || disarmed)
9
+ return false;
10
+ const current = await daemonProcessIdentity(ownership.pid);
11
+ if (disarmed || current.state !== "identified" || current.identity !== ownership.identity)
12
+ return false;
13
+ try {
14
+ return process.kill(ownership.pid, signal);
15
+ }
16
+ catch {
17
+ return false;
18
+ }
19
+ }
20
+ async function orphanCleanup() {
21
+ if (cleanupStarted || disarmed)
22
+ return;
23
+ cleanupStarted = true;
24
+ // Disconnect can arrive while the initial generation check is in flight.
25
+ if (arming)
26
+ await arming;
27
+ let terminatedAt;
28
+ while (ownership && !disarmed) {
29
+ const current = await daemonProcessIdentity(ownership.pid);
30
+ if (current.state === "dead" || (current.state === "identified" && current.identity !== ownership.identity))
31
+ break;
32
+ if (current.state === "identified") {
33
+ // Retry CONT as well: a transient failed probe must not leave TERM
34
+ // queued forever on a stopped process. Every signal checks generation.
35
+ if (process.platform !== "win32")
36
+ await signalOwned("SIGCONT");
37
+ if (terminatedAt === undefined) {
38
+ if (await signalOwned("SIGTERM"))
39
+ terminatedAt = Date.now();
40
+ }
41
+ else if (Date.now() - terminatedAt >= ownership.graceMs) {
42
+ await signalOwned("SIGKILL");
43
+ }
44
+ }
45
+ // Unknown is not permission to signal, nor proof of cleanup. Keep this
46
+ // never-paused, secret-free watchdog alive until identity recovers or the
47
+ // exact generation is positively gone. Each OS lookup is itself bounded.
48
+ await new Promise((resolve) => setTimeout(resolve, 100));
49
+ }
50
+ process.exit(0);
51
+ }
52
+ process.on("message", (wire) => {
53
+ if (!wire || typeof wire !== "object" || !("type" in wire))
54
+ return;
55
+ const message = wire;
56
+ if (message.type === "disarm") {
57
+ // Only the owner sends this, after observing the actual worker exit.
58
+ disarmed = true;
59
+ process.exit(0);
60
+ }
61
+ if (message.type !== "arm" || arming || ownership)
62
+ return;
63
+ if (!Number.isInteger(message.pid) || message.pid <= 0
64
+ || message.pid === process.pid || message.pid === process.ppid
65
+ || typeof message.identity !== "string" || !message.identity
66
+ || typeof message.graceMs !== "number" || !Number.isFinite(message.graceMs)) {
67
+ process.exit(1);
68
+ }
69
+ ownership = {
70
+ pid: message.pid,
71
+ identity: message.identity,
72
+ graceMs: Math.max(1, Math.min(2_000, message.graceMs)),
73
+ };
74
+ arming = (async () => {
75
+ const current = await daemonProcessIdentity(ownership.pid);
76
+ return current.state === "identified" && current.identity === ownership.identity;
77
+ })();
78
+ void arming.then((verified) => {
79
+ if (disconnected || disarmed)
80
+ return;
81
+ if (!verified) {
82
+ process.exit(1);
83
+ return;
84
+ }
85
+ process.send?.({ type: "armed" }, (error) => {
86
+ if (error)
87
+ void orphanCleanup();
88
+ });
89
+ });
90
+ });
91
+ process.once("disconnect", () => {
92
+ disconnected = true;
93
+ void orphanCleanup();
94
+ });
@@ -0,0 +1,21 @@
1
+ import { buildContentIndex, type PublishLocalContentIndexCheckpointOptions } from "./search.js";
2
+ import { type LegacyIndexPublicationOutcome } from "./index-upgrade-publication.js";
3
+ export type BackgroundIndexWorkOptions = NonNullable<Parameters<typeof buildContentIndex>[2]> & {
4
+ /** Cooperative work hook for the pre-build streaming upgrade inspection. */
5
+ upgradeYield?: PublishLocalContentIndexCheckpointOptions["onYield"];
6
+ /** Ciphertext byte progress for the pre-build managed publication. */
7
+ upgradeProgress?: PublishLocalContentIndexCheckpointOptions["onProgress"];
8
+ };
9
+ /** The daemon only needs incremental progress; checkpoint details remain on disk. */
10
+ export type BackgroundIndexWorkResult = {
11
+ indexedNow: number;
12
+ skipped: number;
13
+ };
14
+ type BackgroundIndexWorkDependencies = {
15
+ publishUpgrade(dataDir: string, phrase: string, options: PublishLocalContentIndexCheckpointOptions): Promise<LegacyIndexPublicationOutcome>;
16
+ needsWork(dataDir: string): Promise<boolean>;
17
+ build: typeof buildContentIndex;
18
+ };
19
+ /** The one operation executed by the daemon-owned historical-index child. */
20
+ export declare function runBackgroundIndexWork(dataDir: string, phrase: string, options: BackgroundIndexWorkOptions, dependencies?: Partial<BackgroundIndexWorkDependencies>): Promise<BackgroundIndexWorkResult>;
21
+ export {};
@@ -0,0 +1,25 @@
1
+ import { buildContentIndex, contentIndexNeedsPolicyUpgrade, } from "./search.js";
2
+ import { publishLegacyLocalIndexCheckpointIfNeeded, } from "./index-upgrade-publication.js";
3
+ const defaults = {
4
+ publishUpgrade: publishLegacyLocalIndexCheckpointIfNeeded,
5
+ needsWork: contentIndexNeedsPolicyUpgrade,
6
+ build: buildContentIndex,
7
+ };
8
+ /** The one operation executed by the daemon-owned historical-index child. */
9
+ export async function runBackgroundIndexWork(dataDir, phrase, options, dependencies = {}) {
10
+ const deps = { ...defaults, ...dependencies };
11
+ const upgrade = await deps.publishUpgrade(dataDir, phrase, {
12
+ signal: options.signal,
13
+ remoteIdleMs: options.scheduling?.remoteIdleMs,
14
+ maxBytesPerSecond: options.scheduling?.backgroundTransferBytesPerSecond,
15
+ onYield: options.upgradeYield,
16
+ onProgress: options.upgradeProgress,
17
+ });
18
+ if (upgrade.status === "published" && !await deps.needsWork(dataDir)) {
19
+ return {
20
+ indexedNow: 0,
21
+ skipped: 0,
22
+ };
23
+ }
24
+ return deps.build(dataDir, phrase, options);
25
+ }