sealkeep 0.8.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +163 -14
- package/CHANGELOG.md +244 -1
- package/CONTROL_PLANE.md +2 -2
- package/LICENSE +1 -1
- package/README.md +108 -22
- package/THIRD_PARTY.md +2 -2
- package/THREAT_MODEL.md +23 -4
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/chunk-access.d.ts +26 -4
- package/dist/packages/sealkeep-crypto/src/chunk-access.js +219 -0
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/envelope.d.ts +11 -1
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/envelope.js +58 -21
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/format.d.ts +2 -2
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/index.d.ts +1 -0
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/index.js +1 -0
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/recipients.js +4 -4
- package/dist/packages/sealkeep-crypto/src/squeeze.d.ts +6 -0
- package/dist/packages/sealkeep-crypto/src/squeeze.js +39 -0
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/stream.d.ts +36 -3
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/stream.js +196 -37
- package/dist/site/index.html +1808 -1904
- package/dist/site/llms.txt +67 -0
- package/dist/site/trust/architecture-data-flow.html +53 -0
- package/dist/site/trust/audit-roadmap.html +37 -0
- package/dist/site/trust/deployment-responsibility.html +11 -0
- package/dist/site/trust/dpa-sample.html +30 -0
- package/dist/site/trust/release-provenance.html +21 -0
- package/dist/site/trust/subprocessors.html +15 -0
- package/dist/site/trust/threat-model.html +34 -0
- package/dist/site/trust/toms.html +41 -0
- package/dist/site/trust-document.css +32 -0
- package/dist/site/trust.html +73 -0
- package/dist/site/visual/assets/index-8Nxnkc7o.js +207 -0
- package/dist/site/visual/assets/index-oxLy1bpy.css +1 -0
- package/dist/site/visual/index.html +18 -0
- package/dist/site.zip +0 -0
- package/dist/src/activity.d.ts +9 -0
- package/dist/src/activity.js +90 -1
- package/dist/src/adapters.d.ts +175 -5
- package/dist/src/adapters.js +961 -73
- package/dist/src/agent-context.d.ts +135 -0
- package/dist/src/agent-context.js +1059 -0
- package/dist/src/archive-copies.d.ts +47 -0
- package/dist/src/archive-copies.js +179 -0
- package/dist/src/audit.d.ts +1 -1
- package/dist/src/audit.js +29 -4
- package/dist/src/autopilot.d.ts +52 -7
- package/dist/src/autopilot.js +143 -25
- package/dist/src/background-bandwidth.d.ts +46 -0
- package/dist/src/background-bandwidth.js +301 -0
- package/dist/src/background-cpu.d.ts +82 -0
- package/dist/src/background-cpu.js +212 -0
- package/dist/src/background-worker-error.d.ts +12 -0
- package/dist/src/background-worker-error.js +18 -0
- package/dist/src/branding.d.ts +24 -1
- package/dist/src/branding.js +31 -1
- package/dist/src/bridge.d.ts +233 -0
- package/dist/src/bridge.js +604 -0
- package/dist/src/byte-stream.d.ts +91 -0
- package/dist/src/byte-stream.js +385 -0
- package/dist/src/chunk-store.d.ts +41 -8
- package/dist/src/chunk-store.js +161 -65
- package/dist/src/cli.js +1746 -166
- package/dist/src/cloud.d.ts +841 -31
- package/dist/src/cloud.js +3196 -277
- package/dist/src/context-background.d.ts +37 -0
- package/dist/src/context-background.js +309 -0
- package/dist/src/context-drain-child.d.ts +1 -0
- package/dist/src/context-drain-child.js +98 -0
- package/dist/src/context-reader.d.ts +118 -0
- package/dist/src/context-reader.js +447 -0
- package/dist/src/control-plane/auth.d.ts +32 -4
- package/dist/src/control-plane/auth.js +85 -24
- package/dist/src/control-plane/server.js +19 -6
- package/dist/src/control-plane.d.ts +17 -1
- package/dist/src/control-plane.js +32 -6
- package/dist/src/crypto.d.ts +1 -1
- package/dist/src/crypto.js +5 -5
- package/dist/src/daemon-lease.d.ts +70 -0
- package/dist/src/daemon-lease.js +420 -0
- package/dist/src/daemon.d.ts +94 -1
- package/dist/src/daemon.js +1082 -105
- package/dist/src/darwin-service-policy.d.ts +41 -0
- package/dist/src/darwin-service-policy.js +60 -0
- package/dist/src/dashboard-cli.js +15 -15
- package/dist/src/device-authorization.d.ts +37 -0
- package/dist/src/device-authorization.js +199 -0
- package/dist/src/device-enrollment.d.ts +91 -0
- package/dist/src/device-enrollment.js +349 -0
- package/dist/src/disk.d.ts +17 -12
- package/dist/src/disk.js +43 -17
- package/dist/src/doctor.d.ts +35 -1
- package/dist/src/doctor.js +316 -41
- package/dist/src/durable-ticket-lock.d.ts +24 -0
- package/dist/src/durable-ticket-lock.js +232 -0
- package/dist/src/enroll.d.ts +1 -1
- package/dist/src/enroll.js +13 -7
- package/dist/src/env.d.ts +10 -1
- package/dist/src/env.js +11 -3
- package/dist/src/errors.d.ts +8 -8
- package/dist/src/errors.js +6 -6
- package/dist/src/flush.d.ts +12 -0
- package/dist/src/flush.js +37 -0
- package/dist/src/heartbeat.d.ts +86 -12
- package/dist/src/heartbeat.js +415 -29
- package/dist/src/index-background-watchdog.d.ts +1 -0
- package/dist/src/index-background-watchdog.js +94 -0
- package/dist/src/index-background-work.d.ts +21 -0
- package/dist/src/index-background-work.js +25 -0
- package/dist/src/index-background.d.ts +64 -0
- package/dist/src/index-background.js +394 -0
- package/dist/src/index-build-child.d.ts +1 -0
- package/dist/src/index-build-child.js +109 -0
- package/dist/src/index-manifest.d.ts +52 -0
- package/dist/src/index-manifest.js +444 -0
- package/dist/src/index-publication-proof.d.ts +84 -0
- package/dist/src/index-publication-proof.js +380 -0
- package/dist/src/index-publication-state.d.ts +149 -0
- package/dist/src/index-publication-state.js +696 -0
- package/dist/src/index-publication-verifier.d.ts +89 -0
- package/dist/src/index-publication-verifier.js +341 -0
- package/dist/src/index-publish.d.ts +62 -0
- package/dist/src/index-publish.js +540 -0
- package/dist/src/index-scratch-cleanup.d.ts +19 -0
- package/dist/src/index-scratch-cleanup.js +166 -0
- package/dist/src/index-segment-types.d.ts +132 -0
- package/dist/src/index-segment-types.js +21 -0
- package/dist/src/index-segments.d.ts +9 -0
- package/dist/src/index-segments.js +516 -0
- package/dist/src/index-store.d.ts +123 -0
- package/dist/src/index-store.js +495 -0
- package/dist/src/index-sync.d.ts +91 -15
- package/dist/src/index-sync.js +286 -53
- package/dist/src/index-upgrade-publication.d.ts +30 -0
- package/dist/src/index-upgrade-publication.js +179 -0
- package/dist/src/integration-manager.d.ts +32 -0
- package/dist/src/integration-manager.js +394 -0
- package/dist/src/leakscan.js +1 -1
- package/dist/src/local-api.d.ts +56 -15
- package/dist/src/local-api.js +4979 -541
- package/dist/src/machine-settings.d.ts +51 -0
- package/dist/src/machine-settings.js +166 -0
- package/dist/src/managed-chunks.d.ts +5 -2
- package/dist/src/managed-chunks.js +14 -14
- package/dist/src/mcp-install.d.ts +11 -9
- package/dist/src/mcp-install.js +73 -28
- package/dist/src/mcp-workspace.d.ts +18 -0
- package/dist/src/mcp-workspace.js +50 -0
- package/dist/src/mcp.js +294 -25
- package/dist/src/migrate.d.ts +1 -0
- package/dist/src/migrate.js +33 -5
- package/dist/src/notify.d.ts +1 -1
- package/dist/src/notify.js +5 -5
- package/dist/src/offload.d.ts +201 -14
- package/dist/src/offload.js +1848 -140
- package/dist/src/onboarding.d.ts +8 -1
- package/dist/src/onboarding.js +4 -4
- package/dist/src/packages.d.ts +2 -2
- package/dist/src/packages.js +10 -2
- package/dist/src/passkey.d.ts +0 -1
- package/dist/src/passkey.js +2 -7
- package/dist/src/password-lock.d.ts +2 -2
- package/dist/src/password-lock.js +6 -6
- package/dist/src/paths.d.ts +2 -0
- package/dist/src/paths.js +2 -0
- package/dist/src/presence.d.ts +86 -0
- package/dist/src/presence.js +240 -0
- package/dist/src/progress-deadline.d.ts +21 -0
- package/dist/src/progress-deadline.js +91 -0
- package/dist/src/project-repair.d.ts +55 -0
- package/dist/src/project-repair.js +131 -0
- package/dist/src/providers/gcs.d.ts +28 -7
- package/dist/src/providers/gcs.js +35 -24
- package/dist/src/providers/gdrive.d.ts +71 -8
- package/dist/src/providers/gdrive.js +223 -50
- package/dist/src/providers/index.d.ts +11 -3
- package/dist/src/providers/index.js +30 -10
- package/dist/src/providers/s3.d.ts +30 -8
- package/dist/src/providers/s3.js +41 -30
- package/dist/src/providers/safe-storage-fetch.d.ts +12 -0
- package/dist/src/providers/safe-storage-fetch.js +72 -0
- package/dist/src/queue.d.ts +136 -19
- package/dist/src/queue.js +862 -96
- package/dist/src/reclaim-transaction.d.ts +156 -0
- package/dist/src/reclaim-transaction.js +1027 -0
- package/dist/src/recovery-codes.d.ts +32 -0
- package/dist/src/recovery-codes.js +338 -0
- package/dist/src/recovery.js +12 -9
- package/dist/src/rehydrate.d.ts +25 -22
- package/dist/src/rehydrate.js +319 -23
- package/dist/src/restore.d.ts +57 -4
- package/dist/src/restore.js +272 -36
- package/dist/src/resume-stub.d.ts +92 -0
- package/dist/src/resume-stub.js +417 -0
- package/dist/src/retention.d.ts +98 -7
- package/dist/src/retention.js +1066 -62
- package/dist/src/rotate.js +3 -3
- package/dist/src/search.d.ts +566 -8
- package/dist/src/search.js +5897 -290
- package/dist/src/secrets.d.ts +51 -7
- package/dist/src/secrets.js +316 -24
- package/dist/src/service.d.ts +49 -11
- package/dist/src/service.js +776 -35
- package/dist/src/share.js +3 -3
- package/dist/src/shared-spaces.d.ts +98 -0
- package/dist/src/shared-spaces.js +214 -0
- package/dist/src/source-reader.d.ts +73 -0
- package/dist/src/source-reader.js +715 -0
- package/dist/src/spool.d.ts +1 -1
- package/dist/src/spool.js +1 -1
- package/dist/src/start-tui.js +2 -1
- package/dist/src/start.js +2 -2
- package/dist/src/storage-endpoint.d.ts +21 -0
- package/dist/src/storage-endpoint.js +122 -0
- package/dist/src/storage-setup.js +12 -12
- package/dist/src/storage-targets.d.ts +109 -6
- package/dist/src/storage-targets.js +975 -67
- package/dist/src/stream-to-cloud.d.ts +5 -1
- package/dist/src/stream-to-cloud.js +34 -14
- package/dist/src/sync-rules.d.ts +31 -6
- package/dist/src/sync-rules.js +153 -14
- package/dist/src/team-backfill-scheduling.d.ts +8 -0
- package/dist/src/team-backfill-scheduling.js +33 -0
- package/dist/src/team-backfill.d.ts +116 -0
- package/dist/src/team-backfill.js +1429 -0
- package/dist/src/team-index-cache.d.ts +16 -0
- package/dist/src/team-index-cache.js +152 -0
- package/dist/src/team-offboarding.d.ts +38 -0
- package/dist/src/team-offboarding.js +1043 -0
- package/dist/src/team-presence.d.ts +127 -0
- package/dist/src/team-presence.js +904 -0
- package/dist/src/team-publication-policy.d.ts +20 -0
- package/dist/src/team-publication-policy.js +140 -0
- package/dist/src/team-realtime.d.ts +68 -0
- package/dist/src/team-realtime.js +816 -0
- package/dist/src/team-source-facts-cache.d.ts +23 -0
- package/dist/src/team-source-facts-cache.js +255 -0
- package/dist/src/trash.d.ts +1 -1
- package/dist/src/trash.js +2 -2
- package/dist/src/tui.js +11 -12
- package/dist/src/types.d.ts +173 -7
- package/dist/src/types.js +20 -0
- package/dist/src/ui-server.d.ts +163 -35
- package/dist/src/ui-server.js +712 -72
- package/dist/src/ui.d.ts +1 -2
- package/dist/src/ui.js +1 -2
- package/dist/src/upload.d.ts +27 -0
- package/dist/src/upload.js +383 -43
- package/dist/src/vault.d.ts +235 -29
- package/dist/src/vault.js +1783 -190
- package/dist/src/watcher.d.ts +7 -1
- package/dist/src/watcher.js +198 -55
- package/dist/src/worker.d.ts +27 -3
- package/dist/src/worker.js +274 -55
- package/package.json +33 -12
- package/scripts/native-reboot-rehearsal.mjs +90 -0
- package/web/app.js +6032 -343
- package/web/bootstrap.js +17 -0
- package/web/index.html +255 -57
- package/web/rail.js +317 -40
- package/web/retention.html +2 -2
- package/web/rules-view.js +188 -16
- package/web/sessions-view.js +485 -62
- package/web/sessions.html +2 -2
- package/web/setup-api.js +152 -29
- package/web/setup-logic.js +68 -9
- package/web/setup.html +113 -44
- package/web/setup.js +604 -71
- package/web/style.css +513 -98
- package/dist/packages/vaultline-crypto/src/chunk-access.js +0 -93
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/aead.d.ts +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/aead.js +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/format.js +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/recipients.d.ts +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/sha256-stream.d.ts +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/sha256-stream.js +0 -0
package/dist/src/offload.js
CHANGED
|
@@ -1,12 +1,56 @@
|
|
|
1
|
-
import { mkdtemp, rm, stat, writeFile } from "node:fs/promises";
|
|
1
|
+
import { link as hardLink, lstat, mkdir, mkdtemp, open, readFile, readdir, realpath, rename, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
|
|
2
|
+
import { openForFlush } from "./flush.js";
|
|
3
|
+
import { constants as fsConstants } from "node:fs";
|
|
4
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
5
|
import { tmpdir } from "node:os";
|
|
3
|
-
import { join } from "node:path";
|
|
4
|
-
import { fail } from "./errors.js";
|
|
6
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
7
|
+
import { fail, SealkeepError } from "./errors.js";
|
|
5
8
|
import { recordAudit } from "./audit.js";
|
|
6
9
|
import { isV2 } from "./types.js";
|
|
7
|
-
import { listArchives, readConfig,
|
|
10
|
+
import { listArchives, mutateArchiveRecord, readConfig, resolveDeltaChain } from "./vault.js";
|
|
8
11
|
import { createActiveLease, uploadClientFromStore } from "./providers/index.js";
|
|
9
|
-
import {
|
|
12
|
+
import { archiveCopies, verifiedCopyDestinationKey } from "./archive-copies.js";
|
|
13
|
+
import { byteStreamWithIdleTimeout, digestChunks, LEGACY_BUFFER_LIMIT_BYTES, oneChunk, writeChunksToFile } from "./byte-stream.js";
|
|
14
|
+
import { classifyDaemonProcessGeneration, daemonProcessIdentity } from "./daemon-lease.js";
|
|
15
|
+
import { acquireDurableTicketLock } from "./durable-ticket-lock.js";
|
|
16
|
+
import { paceBackgroundByteStream } from "./background-bandwidth.js";
|
|
17
|
+
import { validateNoFollowPath } from "./reclaim-transaction.js";
|
|
18
|
+
/** Resolve the immutable location recorded on a copy back to its account target. */
|
|
19
|
+
async function configuredTargetForRemote(dataDir, remote) {
|
|
20
|
+
const config = await readConfig(dataDir);
|
|
21
|
+
const { remoteCopyMatchesTarget, resolveTargets } = await import("./storage-targets.js");
|
|
22
|
+
const targets = await resolveTargets(dataDir);
|
|
23
|
+
const byId = remote.targetId ? targets.find((candidate) => candidate.id === remote.targetId) : undefined;
|
|
24
|
+
if (byId && await remoteCopyMatchesTarget(dataDir, byId, remote))
|
|
25
|
+
return byId;
|
|
26
|
+
for (const candidate of targets) {
|
|
27
|
+
if (candidate === byId)
|
|
28
|
+
continue;
|
|
29
|
+
if (await remoteCopyMatchesTarget(dataDir, candidate, remote))
|
|
30
|
+
return candidate;
|
|
31
|
+
}
|
|
32
|
+
if (config.remoteStorage) {
|
|
33
|
+
const primary = { id: "primary", ...config.remoteStorage };
|
|
34
|
+
if (await remoteCopyMatchesTarget(dataDir, primary, remote))
|
|
35
|
+
return primary;
|
|
36
|
+
}
|
|
37
|
+
return fail("storage_not_configured", `The exact ${remote.provider} destination holding this copy is no longer configured on this machine. A reused target id or matching bucket name is not enough; restore the original endpoint/account route.`);
|
|
38
|
+
}
|
|
39
|
+
async function ownBucketConnection(dataDir, target, legacyOverrides = {}) {
|
|
40
|
+
const { credentialConfigIdForTarget, endpointOverridesForResolvedTarget } = await import("./storage-targets.js");
|
|
41
|
+
const credentialId = await credentialConfigIdForTarget(dataDir, target);
|
|
42
|
+
if (!credentialId)
|
|
43
|
+
fail("signer_not_configured", `Storage target "${target.id}" is not connected on this machine.`);
|
|
44
|
+
const ownOverrides = await endpointOverridesForResolvedTarget(dataDir, target);
|
|
45
|
+
return {
|
|
46
|
+
credentialId,
|
|
47
|
+
// Callers may still provide the setup wizard's old endpoint override, but
|
|
48
|
+
// it is valid only for the synthesized legacy primary.
|
|
49
|
+
overrides: Object.keys(ownOverrides).length > 0
|
|
50
|
+
? ownOverrides
|
|
51
|
+
: target.id === "primary" ? legacyOverrides : {},
|
|
52
|
+
};
|
|
53
|
+
}
|
|
10
54
|
/** Providers this build can fetch back from. GCS uploads but has no download client. */
|
|
11
55
|
// Every provider with a proven read-back path. "vaultline" reads back through
|
|
12
56
|
// the account (pullCiphertext, hash-checked); "gcs" through a signed GET on
|
|
@@ -15,31 +59,145 @@ import { sha256 } from "./crypto.js";
|
|
|
15
59
|
// gdrive joined when its client grew download() — the live Drive drill hit
|
|
16
60
|
// this stale gate first: uploads landed, the local copy could never leave.
|
|
17
61
|
const FETCHABLE = new Set(["s3", "r2", "b2", "gcs", "gdrive", "vaultline"]);
|
|
18
|
-
|
|
62
|
+
/**
|
|
63
|
+
* The local-cache clock starts only after both facts are true: the archive
|
|
64
|
+
* exists and an off-machine copy has been verified. An old archive uploaded
|
|
65
|
+
* today therefore receives the same complete cache period as a new one.
|
|
66
|
+
*/
|
|
67
|
+
export function archiveCacheReferenceTime(record) {
|
|
68
|
+
const created = Date.parse(record.createdAt);
|
|
69
|
+
const verified = isV2(record)
|
|
70
|
+
? archiveCopies(record)
|
|
71
|
+
.filter((copy) => FETCHABLE.has(copy.provider))
|
|
72
|
+
.map((copy) => Date.parse(copy.verifiedAt))
|
|
73
|
+
.filter(Number.isFinite)
|
|
74
|
+
: [];
|
|
75
|
+
return Math.max(Number.isFinite(created) ? created : Number.NEGATIVE_INFINITY, ...verified);
|
|
76
|
+
}
|
|
77
|
+
function passesAgeScope(record, options) {
|
|
78
|
+
if (options.olderThanDays === undefined)
|
|
79
|
+
return true;
|
|
80
|
+
if (!Number.isFinite(options.olderThanDays) || options.olderThanDays < 0)
|
|
81
|
+
return false;
|
|
82
|
+
const reference = archiveCacheReferenceTime(record);
|
|
83
|
+
return Number.isFinite(reference)
|
|
84
|
+
&& reference <= (options.now ?? Date.now()) - options.olderThanDays * 86_400_000;
|
|
85
|
+
}
|
|
86
|
+
function passesOffloadScope(record, options) {
|
|
87
|
+
return (!options.only || options.only.has(record.id)) && passesAgeScope(record, options);
|
|
88
|
+
}
|
|
89
|
+
function physicalBytesOf(info) {
|
|
90
|
+
if (typeof info.nlink === "number" && info.nlink > 1)
|
|
91
|
+
return 0;
|
|
92
|
+
if (typeof info.blocks === "number" && Number.isSafeInteger(info.blocks) && info.blocks >= 0) {
|
|
93
|
+
return info.blocks * 512;
|
|
94
|
+
}
|
|
95
|
+
// Node does not expose st_blocks on every platform. Logical size is the
|
|
96
|
+
// least misleading fallback there; POSIX callers always take the branch
|
|
97
|
+
// above, including sparse files whose physical allocation is genuinely 0.
|
|
98
|
+
return info.size;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Returns one destination shared by every link in a delta chain, but only when
|
|
102
|
+
* every link at that destination is readable by this build. A verified copy
|
|
103
|
+
* on each link is not enough: split A/B copies cannot restore the session.
|
|
104
|
+
*/
|
|
105
|
+
function commonReadableDestination(chain) {
|
|
106
|
+
if (chain.length === 1) {
|
|
107
|
+
return archiveCopies(chain[0]).some((copy) => Number.isFinite(Date.parse(copy.verifiedAt)) && FETCHABLE.has(copy.provider))
|
|
108
|
+
? "self-contained"
|
|
109
|
+
: null;
|
|
110
|
+
}
|
|
111
|
+
let common = null;
|
|
112
|
+
for (const record of chain) {
|
|
113
|
+
const destinations = new Set(archiveCopies(record)
|
|
114
|
+
.filter((copy) => Number.isFinite(Date.parse(copy.verifiedAt)) && FETCHABLE.has(copy.provider))
|
|
115
|
+
.map(verifiedCopyDestinationKey)
|
|
116
|
+
.filter((key) => Boolean(key)));
|
|
117
|
+
if (common === null)
|
|
118
|
+
common = destinations;
|
|
119
|
+
else {
|
|
120
|
+
const previous = common;
|
|
121
|
+
common = new Set([...previous].filter((key) => destinations.has(key)));
|
|
122
|
+
}
|
|
123
|
+
if (common.size === 0)
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
return [...(common ?? [])][0] ?? null;
|
|
127
|
+
}
|
|
128
|
+
export async function offloadable(dataDir, options = {}) {
|
|
19
129
|
const archives = await listArchives(dataDir);
|
|
20
130
|
const candidates = [];
|
|
21
131
|
const blocked = [];
|
|
132
|
+
// Resolve the graph once so an incomplete chain blocks every local
|
|
133
|
+
// dependency, and an age-limited sweep never removes an old base still
|
|
134
|
+
// needed by a recent retained head.
|
|
135
|
+
const chainsById = new Map();
|
|
136
|
+
const unsafeDependencyIds = new Set();
|
|
137
|
+
const retainedByRecentHead = new Set();
|
|
138
|
+
for (const item of archives) {
|
|
139
|
+
if (!isV2(item))
|
|
140
|
+
continue;
|
|
141
|
+
let chain;
|
|
142
|
+
try {
|
|
143
|
+
const resolved = resolveDeltaChain(archives, item);
|
|
144
|
+
if (!resolved.every(isV2))
|
|
145
|
+
continue;
|
|
146
|
+
chain = resolved;
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
chainsById.set(item.id, chain);
|
|
152
|
+
if (chain.length > 1 && !commonReadableDestination(chain)) {
|
|
153
|
+
for (const link of chain)
|
|
154
|
+
unsafeDependencyIds.add(link.id);
|
|
155
|
+
}
|
|
156
|
+
if (options.olderThanDays !== undefined && !passesAgeScope(item, options) && chain.length > 1) {
|
|
157
|
+
for (const link of chain)
|
|
158
|
+
retainedByRecentHead.add(link.id);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
22
161
|
for (const record of archives) {
|
|
162
|
+
if (!passesOffloadScope(record, options))
|
|
163
|
+
continue;
|
|
23
164
|
if (!isV2(record)) {
|
|
24
165
|
blocked.push({ id: record.id, reason: "archive uses format v1; run `sealkeep migrate` first" });
|
|
25
166
|
continue;
|
|
26
167
|
}
|
|
27
168
|
if (record.offloaded)
|
|
28
169
|
continue;
|
|
29
|
-
|
|
30
|
-
if (!remote?.verifiedAt) {
|
|
31
|
-
blocked.push({ id: record.id, reason: "no verified remote copy, so the local one is the only copy" });
|
|
32
|
-
continue;
|
|
33
|
-
}
|
|
34
|
-
if (!FETCHABLE.has(remote.provider)) {
|
|
35
|
-
blocked.push({ id: record.id, reason: `${remote.provider} can be uploaded to but not read back by this build, so the local copy has to stay` });
|
|
170
|
+
if (retainedByRecentHead.has(record.id))
|
|
36
171
|
continue;
|
|
37
|
-
}
|
|
38
172
|
const local = await stat(record.objectPath).catch(() => null);
|
|
39
173
|
if (!local) {
|
|
40
174
|
blocked.push({ id: record.id, reason: "no local archive file to remove" });
|
|
41
175
|
continue;
|
|
42
176
|
}
|
|
177
|
+
const chain = chainsById.get(record.id);
|
|
178
|
+
if (!chain) {
|
|
179
|
+
blocked.push({ id: record.id, reason: "the archive's delta dependency chain is incomplete, so no safe offload proof exists" });
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
if (unsafeDependencyIds.has(record.id)) {
|
|
183
|
+
blocked.push({ id: record.id, reason: "verified copies for this delta chain are split or not readable at one destination, so the local copy stays" });
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
const verified = archiveCopies(record).filter((copy) => Number.isFinite(Date.parse(copy.verifiedAt)));
|
|
187
|
+
if (verified.length === 0) {
|
|
188
|
+
blocked.push({ id: record.id, reason: "no verified remote copy, so the local archive is the only copy" });
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
const readable = verified.filter((copy) => FETCHABLE.has(copy.provider));
|
|
192
|
+
if (readable.length === 0) {
|
|
193
|
+
blocked.push({ id: record.id, reason: `${verified.map((copy) => copy.provider).join(", ")} cannot be read back by this build, so the local copy has to stay` });
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
const commonDestination = commonReadableDestination(chain);
|
|
197
|
+
if (!commonDestination) {
|
|
198
|
+
blocked.push({ id: record.id, reason: "verified copies for this delta chain are split or not readable at one destination, so the local copy stays" });
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
43
201
|
// The remote is only a copy if the sizes agree. For a provider that stores
|
|
44
202
|
// the raw ciphertext that means equality; a managed object carries the
|
|
45
203
|
// self-opening envelope frame around the same ciphertext, so it is larger
|
|
@@ -47,17 +205,678 @@ export async function offloadable(dataDir) {
|
|
|
47
205
|
// demanding it silently blocked every managed offload. The frame only
|
|
48
206
|
// adds, so smaller-than-local is still the same corruption signal, and
|
|
49
207
|
// the true proof either way is the fetch-and-hash below.
|
|
50
|
-
const
|
|
51
|
-
|
|
52
|
-
|
|
208
|
+
const remote = readable.find((copy) => (chain.length === 1 || verifiedCopyDestinationKey(copy) === commonDestination)
|
|
209
|
+
&& (copy.provider === "vaultline" ? copy.bytes >= local.size : copy.bytes === local.size));
|
|
210
|
+
if (!remote) {
|
|
211
|
+
blocked.push({ id: record.id, reason: `none of the ${readable.length} readable stored copies has the expected ${local.size} bytes` });
|
|
53
212
|
continue;
|
|
54
213
|
}
|
|
55
214
|
candidates.push({
|
|
56
215
|
id: record.id, agent: record.source.agent, path: record.source.path,
|
|
57
|
-
localBytes: local.size,
|
|
216
|
+
localBytes: local.size, allocatedBytes: physicalBytesOf(local),
|
|
217
|
+
provider: remote.provider, bucket: remote.bucket, objectKey: remote.objectKey,
|
|
218
|
+
copyId: remote.copyId
|
|
58
219
|
});
|
|
59
220
|
}
|
|
60
|
-
return { candidates, blocked, freeableBytes: candidates.reduce((total, item) => total + item.
|
|
221
|
+
return { candidates, blocked, freeableBytes: candidates.reduce((total, item) => total + item.allocatedBytes, 0) };
|
|
222
|
+
}
|
|
223
|
+
const OFFLOAD_STAGES = new Set(["prepared", "linked", "remote-verified", "metadata-committed", "source-unlinked", "purge-authorized"]);
|
|
224
|
+
const OFFLOAD_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
225
|
+
function offloadJournalDirectory(dataDir) {
|
|
226
|
+
return join(resolve(dataDir), "offload-transactions");
|
|
227
|
+
}
|
|
228
|
+
export function offloadJournalPath(dataDir, id) {
|
|
229
|
+
if (!OFFLOAD_ID.test(id))
|
|
230
|
+
throw new Error("Invalid offload transaction id");
|
|
231
|
+
return join(offloadJournalDirectory(dataDir), `${id}.json`);
|
|
232
|
+
}
|
|
233
|
+
function offloadStagingPath(objectPath, id) {
|
|
234
|
+
return join(dirname(resolve(objectPath)), `.${basename(objectPath)}.sealkeep-offload-${id}.staged`);
|
|
235
|
+
}
|
|
236
|
+
async function syncOffloadDirectory(path) {
|
|
237
|
+
let handle;
|
|
238
|
+
try {
|
|
239
|
+
handle = await open(path, "r");
|
|
240
|
+
await handle.sync();
|
|
241
|
+
}
|
|
242
|
+
catch (error) {
|
|
243
|
+
if (process.platform !== "win32")
|
|
244
|
+
throw error;
|
|
245
|
+
}
|
|
246
|
+
finally {
|
|
247
|
+
await handle?.close();
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
async function canonicalNonSymlinkDirectory(path, label) {
|
|
251
|
+
const lexical = resolve(path);
|
|
252
|
+
const info = await lstat(lexical);
|
|
253
|
+
if (!info.isDirectory() || info.isSymbolicLink())
|
|
254
|
+
throw new Error(`${label} is not a non-symlink directory: ${lexical}`);
|
|
255
|
+
const canonical = await realpath(lexical);
|
|
256
|
+
const canonicalInfo = await lstat(canonical);
|
|
257
|
+
if (!canonicalInfo.isDirectory() || canonicalInfo.isSymbolicLink())
|
|
258
|
+
throw new Error(`${label} does not resolve to a trusted directory: ${lexical}`);
|
|
259
|
+
return canonical;
|
|
260
|
+
}
|
|
261
|
+
async function trustedChildDirectory(parent, name, create) {
|
|
262
|
+
const path = join(parent, name);
|
|
263
|
+
let created = false;
|
|
264
|
+
if (create) {
|
|
265
|
+
try {
|
|
266
|
+
await mkdir(path, { mode: 0o700 });
|
|
267
|
+
created = true;
|
|
268
|
+
}
|
|
269
|
+
catch (error) {
|
|
270
|
+
if (error.code !== "EEXIST")
|
|
271
|
+
throw error;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
const info = await lstat(path).catch((error) => {
|
|
275
|
+
if (error.code === "ENOENT" && !create)
|
|
276
|
+
return null;
|
|
277
|
+
throw error;
|
|
278
|
+
});
|
|
279
|
+
if (!info)
|
|
280
|
+
return null;
|
|
281
|
+
if (!info.isDirectory() || info.isSymbolicLink() || await realpath(path) !== path) {
|
|
282
|
+
throw new Error(`Offload transaction directory is not a canonical non-symlink directory: ${path}`);
|
|
283
|
+
}
|
|
284
|
+
if (created) {
|
|
285
|
+
await syncOffloadDirectory(parent);
|
|
286
|
+
await syncOffloadDirectory(path);
|
|
287
|
+
}
|
|
288
|
+
return path;
|
|
289
|
+
}
|
|
290
|
+
async function trustedOffloadJournalDirectory(dataDir, create) {
|
|
291
|
+
const canonicalDataDir = await canonicalNonSymlinkDirectory(dataDir, "Sealkeep data directory");
|
|
292
|
+
return trustedChildDirectory(canonicalDataDir, "offload-transactions", create);
|
|
293
|
+
}
|
|
294
|
+
async function trustedOffloadLockPath(dataDir, key) {
|
|
295
|
+
const journalDirectory = await trustedOffloadJournalDirectory(dataDir, true);
|
|
296
|
+
if (!journalDirectory)
|
|
297
|
+
throw new Error("Could not create the offload transaction directory");
|
|
298
|
+
const lockDirectory = await trustedChildDirectory(journalDirectory, ".locks", true);
|
|
299
|
+
if (!lockDirectory)
|
|
300
|
+
throw new Error("Could not create the offload lock directory");
|
|
301
|
+
const queueDirectory = await trustedChildDirectory(lockDirectory, key, true);
|
|
302
|
+
if (!queueDirectory)
|
|
303
|
+
throw new Error("Could not create the archive offload lock queue");
|
|
304
|
+
return queueDirectory;
|
|
305
|
+
}
|
|
306
|
+
async function assertNoFollowJournal(path) {
|
|
307
|
+
const info = await lstat(path);
|
|
308
|
+
if (!info.isFile() || info.isSymbolicLink())
|
|
309
|
+
throw new Error(`Offload journal is not a non-symlink regular file: ${path}`);
|
|
310
|
+
}
|
|
311
|
+
async function readJournalNoFollow(path) {
|
|
312
|
+
const noFollow = typeof fsConstants.O_NOFOLLOW === "number" ? fsConstants.O_NOFOLLOW : 0;
|
|
313
|
+
const handle = await open(path, fsConstants.O_RDONLY | noFollow);
|
|
314
|
+
try {
|
|
315
|
+
const before = await handle.stat({ bigint: true });
|
|
316
|
+
if (!before.isFile())
|
|
317
|
+
throw new Error(`Offload journal is not a regular file: ${path}`);
|
|
318
|
+
const raw = await handle.readFile("utf8");
|
|
319
|
+
const after = await handle.stat({ bigint: true });
|
|
320
|
+
if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size
|
|
321
|
+
|| before.mtimeNs !== after.mtimeNs || before.ctimeNs !== after.ctimeNs) {
|
|
322
|
+
throw new Error(`Offload journal changed while it was being read: ${path}`);
|
|
323
|
+
}
|
|
324
|
+
return raw;
|
|
325
|
+
}
|
|
326
|
+
finally {
|
|
327
|
+
await handle.close();
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
async function writeOffloadJournal(transaction, exclusive = false) {
|
|
331
|
+
const directory = await trustedOffloadJournalDirectory(transactionDataDir(transaction), true);
|
|
332
|
+
if (!directory)
|
|
333
|
+
throw new Error("Could not create the offload transaction directory");
|
|
334
|
+
const path = join(directory, `${transaction.id}.json`);
|
|
335
|
+
const temporary = join(directory, `.${basename(path)}.${randomUUID()}.tmp`);
|
|
336
|
+
let handle;
|
|
337
|
+
try {
|
|
338
|
+
handle = await open(temporary, "wx", 0o600);
|
|
339
|
+
await handle.writeFile(`${JSON.stringify(transaction)}\n`, "utf8");
|
|
340
|
+
await handle.sync();
|
|
341
|
+
await handle.close();
|
|
342
|
+
handle = undefined;
|
|
343
|
+
if (exclusive)
|
|
344
|
+
await hardLink(temporary, path);
|
|
345
|
+
else {
|
|
346
|
+
await assertNoFollowJournal(path);
|
|
347
|
+
await rename(temporary, path);
|
|
348
|
+
}
|
|
349
|
+
await syncOffloadDirectory(directory);
|
|
350
|
+
}
|
|
351
|
+
finally {
|
|
352
|
+
await handle?.close().catch(() => undefined);
|
|
353
|
+
await rm(temporary, { force: true }).catch(() => undefined);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
async function removeOffloadJournal(dataDir, id) {
|
|
357
|
+
const directory = await trustedOffloadJournalDirectory(dataDir, false);
|
|
358
|
+
if (!directory)
|
|
359
|
+
throw new Error("Offload transaction directory disappeared before journal cleanup");
|
|
360
|
+
const path = join(directory, `${id}.json`);
|
|
361
|
+
await assertNoFollowJournal(path);
|
|
362
|
+
await unlink(path);
|
|
363
|
+
await syncOffloadDirectory(directory);
|
|
364
|
+
}
|
|
365
|
+
function transactionDataDir(transaction) {
|
|
366
|
+
const value = transaction.dataDir;
|
|
367
|
+
if (!value)
|
|
368
|
+
throw new Error("Offload transaction has no owning data directory");
|
|
369
|
+
return value;
|
|
370
|
+
}
|
|
371
|
+
function liveOffloadTransaction(dataDir, value) {
|
|
372
|
+
Object.defineProperty(value, "dataDir", { value: resolve(dataDir), enumerable: false, configurable: false, writable: false });
|
|
373
|
+
return value;
|
|
374
|
+
}
|
|
375
|
+
async function replaceOffloadJournal(transaction, stage, patch = {}) {
|
|
376
|
+
const next = liveOffloadTransaction(transaction.dataDir, {
|
|
377
|
+
...transaction,
|
|
378
|
+
...patch,
|
|
379
|
+
updatedAt: new Date().toISOString(),
|
|
380
|
+
stage,
|
|
381
|
+
});
|
|
382
|
+
await writeOffloadJournal(next);
|
|
383
|
+
return next;
|
|
384
|
+
}
|
|
385
|
+
async function removeOffloadPathDurably(path) {
|
|
386
|
+
await unlink(path);
|
|
387
|
+
await syncOffloadDirectory(dirname(path));
|
|
388
|
+
}
|
|
389
|
+
async function localArchiveIdentity(path) {
|
|
390
|
+
const info = await lstat(path, { bigint: true });
|
|
391
|
+
if (!info.isFile() || info.isSymbolicLink())
|
|
392
|
+
throw new Error(`Local archive is not a regular non-symlink file: ${path}`);
|
|
393
|
+
if (info.size > BigInt(Number.MAX_SAFE_INTEGER) || info.blocks * 512n > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
394
|
+
throw new Error(`Local archive is too large to measure safely: ${path}`);
|
|
395
|
+
}
|
|
396
|
+
return {
|
|
397
|
+
bytes: Number(info.size),
|
|
398
|
+
allocatedBytes: Number(info.blocks * 512n),
|
|
399
|
+
links: Number(info.nlink),
|
|
400
|
+
dev: info.dev.toString(),
|
|
401
|
+
ino: info.ino.toString(),
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
async function localArchiveIdentityOrNull(path) {
|
|
405
|
+
try {
|
|
406
|
+
return await localArchiveIdentity(path);
|
|
407
|
+
}
|
|
408
|
+
catch (error) {
|
|
409
|
+
if (error.code === "ENOENT")
|
|
410
|
+
return null;
|
|
411
|
+
throw error;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
function sameOffloadInode(left, right) {
|
|
415
|
+
return left.dev === right.dev && left.ino === right.ino;
|
|
416
|
+
}
|
|
417
|
+
function sameOffloadFile(left, right) {
|
|
418
|
+
return sameOffloadInode(left, right) && left.bytes === right.bytes;
|
|
419
|
+
}
|
|
420
|
+
function sameOffloadSnapshot(left, right) {
|
|
421
|
+
return sameOffloadFile(left, right)
|
|
422
|
+
&& left.allocatedBytes === right.allocatedBytes
|
|
423
|
+
&& left.links === right.links;
|
|
424
|
+
}
|
|
425
|
+
function parseOffloadJournal(dataDir, name, raw) {
|
|
426
|
+
const value = JSON.parse(raw);
|
|
427
|
+
const original = value.original;
|
|
428
|
+
if (value.version !== 1
|
|
429
|
+
|| typeof value.id !== "string" || !OFFLOAD_ID.test(value.id) || name !== `${value.id}.json`
|
|
430
|
+
|| typeof value.archiveId !== "string" || !OFFLOAD_ID.test(value.archiveId)
|
|
431
|
+
|| typeof value.objectPath !== "string" || value.objectPath !== resolve(value.objectPath)
|
|
432
|
+
|| typeof value.stagingPath !== "string" || value.stagingPath !== offloadStagingPath(value.objectPath, value.id)
|
|
433
|
+
|| typeof value.createdAt !== "string" || !Number.isFinite(Date.parse(value.createdAt))
|
|
434
|
+
|| typeof value.updatedAt !== "string" || !Number.isFinite(Date.parse(value.updatedAt))
|
|
435
|
+
|| !value.stage || !OFFLOAD_STAGES.has(value.stage)
|
|
436
|
+
|| !original || !Number.isSafeInteger(original.bytes) || (original.bytes ?? -1) < 0
|
|
437
|
+
|| !Number.isSafeInteger(original.allocatedBytes) || (original.allocatedBytes ?? -1) < 0
|
|
438
|
+
|| !Number.isSafeInteger(original.links) || (original.links ?? 0) < 1
|
|
439
|
+
|| typeof original.dev !== "string" || typeof original.ino !== "string"
|
|
440
|
+
|| (value.purgeSnapshot !== undefined && (!Number.isSafeInteger(value.purgeSnapshot.bytes) || value.purgeSnapshot.bytes < 0
|
|
441
|
+
|| !Number.isSafeInteger(value.purgeSnapshot.allocatedBytes) || value.purgeSnapshot.allocatedBytes < 0
|
|
442
|
+
|| !Number.isSafeInteger(value.purgeSnapshot.links) || value.purgeSnapshot.links < 1
|
|
443
|
+
|| typeof value.purgeSnapshot.dev !== "string" || typeof value.purgeSnapshot.ino !== "string"))
|
|
444
|
+
|| !value.remote || typeof value.remote.provider !== "string" || typeof value.remote.bucket !== "string" || typeof value.remote.objectKey !== "string"
|
|
445
|
+
|| (value.remote.copyId !== undefined && typeof value.remote.copyId !== "string")
|
|
446
|
+
|| (value.remote.destinationKey !== undefined && typeof value.remote.destinationKey !== "string")) {
|
|
447
|
+
throw new Error(`Invalid offload transaction journal: ${join(offloadJournalDirectory(dataDir), name)}`);
|
|
448
|
+
}
|
|
449
|
+
return liveOffloadTransaction(dataDir, value);
|
|
450
|
+
}
|
|
451
|
+
async function readOffloadJournals(dataDir) {
|
|
452
|
+
const directory = await trustedOffloadJournalDirectory(dataDir, false);
|
|
453
|
+
if (!directory)
|
|
454
|
+
return [];
|
|
455
|
+
const names = await readdir(directory);
|
|
456
|
+
const journals = [];
|
|
457
|
+
for (const name of names.filter((item) => item.endsWith(".json")).sort()) {
|
|
458
|
+
const path = join(directory, name);
|
|
459
|
+
await assertNoFollowJournal(path);
|
|
460
|
+
journals.push(parseOffloadJournal(dataDir, name, await readJournalNoFollow(path)));
|
|
461
|
+
}
|
|
462
|
+
return journals;
|
|
463
|
+
}
|
|
464
|
+
async function beginOffloadTransaction(dataDir, candidate, record) {
|
|
465
|
+
const original = await localArchiveIdentity(record.objectPath);
|
|
466
|
+
if (original.bytes !== record.cipher.storedBytes || original.bytes !== candidate.localBytes) {
|
|
467
|
+
throw new Error("the local encrypted archive changed while offload was starting; local copy kept");
|
|
468
|
+
}
|
|
469
|
+
const now = new Date().toISOString();
|
|
470
|
+
const id = randomUUID();
|
|
471
|
+
const selected = archiveCopies(record).find((copy) => copy.copyId === candidate.copyId);
|
|
472
|
+
if (!selected)
|
|
473
|
+
throw new Error("the selected verified copy changed before offload could start; local copy kept");
|
|
474
|
+
const destinationKey = verifiedCopyDestinationKey(selected);
|
|
475
|
+
const transaction = liveOffloadTransaction(dataDir, {
|
|
476
|
+
version: 1,
|
|
477
|
+
id,
|
|
478
|
+
archiveId: record.id,
|
|
479
|
+
objectPath: resolve(record.objectPath),
|
|
480
|
+
stagingPath: offloadStagingPath(record.objectPath, id),
|
|
481
|
+
createdAt: now,
|
|
482
|
+
updatedAt: now,
|
|
483
|
+
stage: "prepared",
|
|
484
|
+
original,
|
|
485
|
+
remote: {
|
|
486
|
+
...(candidate.copyId ? { copyId: candidate.copyId } : {}),
|
|
487
|
+
...(destinationKey ? { destinationKey } : {}),
|
|
488
|
+
provider: candidate.provider,
|
|
489
|
+
bucket: candidate.bucket,
|
|
490
|
+
objectKey: candidate.objectKey,
|
|
491
|
+
},
|
|
492
|
+
});
|
|
493
|
+
await assertOffloadRecordOwnsPath(dataDir, transaction);
|
|
494
|
+
await writeOffloadJournal(transaction, true);
|
|
495
|
+
return transaction;
|
|
496
|
+
}
|
|
497
|
+
async function linkOffloadStaging(transaction) {
|
|
498
|
+
const before = await localArchiveIdentity(transaction.objectPath);
|
|
499
|
+
if (!sameOffloadFile(before, transaction.original)) {
|
|
500
|
+
throw new Error("the local encrypted archive changed before it could be staged; local copy kept");
|
|
501
|
+
}
|
|
502
|
+
await hardLink(transaction.objectPath, transaction.stagingPath);
|
|
503
|
+
const [source, staged] = await Promise.all([
|
|
504
|
+
localArchiveIdentity(transaction.objectPath),
|
|
505
|
+
localArchiveIdentity(transaction.stagingPath),
|
|
506
|
+
]);
|
|
507
|
+
if (!sameOffloadFile(source, staged) || !sameOffloadFile(source, transaction.original)) {
|
|
508
|
+
throw new Error("the encrypted archive staging link does not identify the original inode; both paths were kept");
|
|
509
|
+
}
|
|
510
|
+
const { handle: stagedHandle } = await openForFlush(transaction.stagingPath);
|
|
511
|
+
try {
|
|
512
|
+
await stagedHandle.sync();
|
|
513
|
+
}
|
|
514
|
+
finally {
|
|
515
|
+
await stagedHandle.close();
|
|
516
|
+
}
|
|
517
|
+
await syncOffloadDirectory(dirname(transaction.stagingPath));
|
|
518
|
+
return replaceOffloadJournal(transaction, "linked");
|
|
519
|
+
}
|
|
520
|
+
function selectedOffloadCopy(record, transaction) {
|
|
521
|
+
return archiveCopies(record).find((copy) => transaction.remote.copyId
|
|
522
|
+
? copy.copyId === transaction.remote.copyId
|
|
523
|
+
: copy.provider === transaction.remote.provider
|
|
524
|
+
&& copy.bucket === transaction.remote.bucket
|
|
525
|
+
&& copy.objectKey === transaction.remote.objectKey) ?? null;
|
|
526
|
+
}
|
|
527
|
+
function offloadMetadataMatches(record, transaction) {
|
|
528
|
+
return record.offloaded?.transactionId === transaction.id
|
|
529
|
+
&& record.offloaded.provider === transaction.remote.provider
|
|
530
|
+
&& record.offloaded.bucket === transaction.remote.bucket
|
|
531
|
+
&& record.offloaded.objectKey === transaction.remote.objectKey;
|
|
532
|
+
}
|
|
533
|
+
function offloadMetadataFinalized(record, transaction) {
|
|
534
|
+
const offloaded = record.offloaded;
|
|
535
|
+
return Boolean(offloaded
|
|
536
|
+
&& offloaded.transactionId === undefined
|
|
537
|
+
&& offloaded.provider === transaction.remote.provider
|
|
538
|
+
&& offloaded.bucket === transaction.remote.bucket
|
|
539
|
+
&& offloaded.objectKey === transaction.remote.objectKey
|
|
540
|
+
&& offloaded.localBytes === transaction.original.bytes
|
|
541
|
+
&& Number.isSafeInteger(offloaded.freedBytes)
|
|
542
|
+
&& (offloaded.freedBytes ?? -1) >= 0);
|
|
543
|
+
}
|
|
544
|
+
async function verifyOffloadRemoteChain(dataDir, record, transaction, options) {
|
|
545
|
+
const selected = selectedOffloadCopy(record, transaction);
|
|
546
|
+
if (!selected)
|
|
547
|
+
throw new Error("the verified restore copy changed while offload was being checked; local copy kept");
|
|
548
|
+
if (!Number.isFinite(Date.parse(selected.verifiedAt)) || !FETCHABLE.has(selected.provider)) {
|
|
549
|
+
throw new Error("the selected restore copy is no longer verified and readable; local copy kept");
|
|
550
|
+
}
|
|
551
|
+
const archives = await listArchives(dataDir);
|
|
552
|
+
const liveRecord = archives.find((item) => item.id === record.id && isV2(item));
|
|
553
|
+
if (!liveRecord)
|
|
554
|
+
throw new Error("the archive disappeared while its remote dependency chain was being checked; local copy kept");
|
|
555
|
+
const resolved = resolveDeltaChain(archives, liveRecord);
|
|
556
|
+
if (!resolved.every(isV2))
|
|
557
|
+
throw new Error("the archive's remote dependency chain is incomplete; local copy kept");
|
|
558
|
+
const chain = resolved;
|
|
559
|
+
const selectedDestination = transaction.remote.destinationKey ?? verifiedCopyDestinationKey(selected);
|
|
560
|
+
if (chain.length > 1 && !selectedDestination) {
|
|
561
|
+
throw new Error("the archive's remote dependency chain has no stable common destination; local copy kept");
|
|
562
|
+
}
|
|
563
|
+
for (const link of chain) {
|
|
564
|
+
const copy = link.id === liveRecord.id
|
|
565
|
+
? selectedOffloadCopy(link, transaction)
|
|
566
|
+
: archiveCopies(link).find((candidate) => Number.isFinite(Date.parse(candidate.verifiedAt))
|
|
567
|
+
&& FETCHABLE.has(candidate.provider)
|
|
568
|
+
&& verifiedCopyDestinationKey(candidate) === selectedDestination) ?? null;
|
|
569
|
+
if (!copy || !Number.isFinite(Date.parse(copy.verifiedAt)) || !FETCHABLE.has(copy.provider)
|
|
570
|
+
|| (chain.length > 1 && verifiedCopyDestinationKey(copy) !== selectedDestination)) {
|
|
571
|
+
throw new Error(`the complete remote delta chain is no longer readable at one destination (${link.id}); local copy kept`);
|
|
572
|
+
}
|
|
573
|
+
let stream = await fetchCiphertextStream(dataDir, link, options.client, copy, options.overrides, options.signal);
|
|
574
|
+
if (options.maxBytesPerSecond !== undefined) {
|
|
575
|
+
stream = paceBackgroundByteStream(dataDir, stream, options.maxBytesPerSecond, { signal: options.signal });
|
|
576
|
+
}
|
|
577
|
+
const fetched = await digestChunks(stream);
|
|
578
|
+
if (fetched.sha256Hex !== link.cipher.ciphertextSha256 || fetched.bytes !== link.cipher.storedBytes) {
|
|
579
|
+
throw new Error(`what the bucket returned is not this archive (${link.id}) in the required delta chain; local copy kept`);
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
async function markOffloadMetadata(dataDir, transaction) {
|
|
584
|
+
const config = await readConfig(dataDir);
|
|
585
|
+
await mutateArchiveRecord(config.storage.root, transaction.archiveId, (current) => {
|
|
586
|
+
if (!isV2(current) || resolve(current.objectPath) !== transaction.objectPath || !selectedOffloadCopy(current, transaction)) {
|
|
587
|
+
throw new Error("archive metadata changed while offload was being committed; local copy kept");
|
|
588
|
+
}
|
|
589
|
+
if (current.offloaded && !offloadMetadataMatches(current, transaction)) {
|
|
590
|
+
throw new Error("another offload already changed this archive record; local copy kept");
|
|
591
|
+
}
|
|
592
|
+
return {
|
|
593
|
+
...current,
|
|
594
|
+
offloaded: {
|
|
595
|
+
at: current.offloaded?.at ?? new Date().toISOString(),
|
|
596
|
+
provider: transaction.remote.provider,
|
|
597
|
+
bucket: transaction.remote.bucket,
|
|
598
|
+
objectKey: transaction.remote.objectKey,
|
|
599
|
+
transactionId: transaction.id,
|
|
600
|
+
localBytes: transaction.original.bytes,
|
|
601
|
+
freedBytes: 0,
|
|
602
|
+
},
|
|
603
|
+
};
|
|
604
|
+
});
|
|
605
|
+
return replaceOffloadJournal(transaction, "metadata-committed");
|
|
606
|
+
}
|
|
607
|
+
async function unlinkOffloadSource(transaction) {
|
|
608
|
+
const [source, staged] = await Promise.all([
|
|
609
|
+
localArchiveIdentity(transaction.objectPath),
|
|
610
|
+
localArchiveIdentity(transaction.stagingPath),
|
|
611
|
+
]);
|
|
612
|
+
if (!sameOffloadFile(source, staged) || !sameOffloadFile(source, transaction.original)) {
|
|
613
|
+
throw new Error("the local archive path changed before offload could remove it; every file was kept");
|
|
614
|
+
}
|
|
615
|
+
await removeOffloadPathDurably(transaction.objectPath);
|
|
616
|
+
return replaceOffloadJournal(transaction, "source-unlinked");
|
|
617
|
+
}
|
|
618
|
+
async function finalizeOffloadMetadata(dataDir, transaction, freedBytes) {
|
|
619
|
+
const config = await readConfig(dataDir);
|
|
620
|
+
await mutateArchiveRecord(config.storage.root, transaction.archiveId, (current) => {
|
|
621
|
+
if (!isV2(current) || !offloadMetadataMatches(current, transaction)) {
|
|
622
|
+
throw new Error("offload completed but its archive record changed; the recovery journal was retained");
|
|
623
|
+
}
|
|
624
|
+
const { transactionId: _transactionId, ...offloaded } = current.offloaded;
|
|
625
|
+
return { ...current, offloaded: { ...offloaded, freedBytes, localBytes: transaction.original.bytes } };
|
|
626
|
+
});
|
|
627
|
+
await removeOffloadJournal(dataDir, transaction.id);
|
|
628
|
+
return freedBytes;
|
|
629
|
+
}
|
|
630
|
+
async function clearOwnedOffloadMetadata(dataDir, transaction) {
|
|
631
|
+
const config = await readConfig(dataDir);
|
|
632
|
+
await mutateArchiveRecord(config.storage.root, transaction.archiveId, (current) => {
|
|
633
|
+
if (!isV2(current) || !offloadMetadataMatches(current, transaction))
|
|
634
|
+
return current;
|
|
635
|
+
const { offloaded: _offloaded, ...rest } = current;
|
|
636
|
+
return rest;
|
|
637
|
+
});
|
|
638
|
+
}
|
|
639
|
+
async function assertOffloadRecordOwnsPath(dataDir, transaction) {
|
|
640
|
+
const config = await readConfig(dataDir);
|
|
641
|
+
const record = (await listArchives(dataDir)).find((item) => item.id === transaction.archiveId && isV2(item));
|
|
642
|
+
const lexicalRoot = resolve(config.storage.root);
|
|
643
|
+
const expectedObject = join(lexicalRoot, `${transaction.archiveId}.skarchive`);
|
|
644
|
+
if (!record || resolve(record.objectPath) !== transaction.objectPath
|
|
645
|
+
|| transaction.objectPath !== expectedObject
|
|
646
|
+
|| dirname(transaction.stagingPath) !== lexicalRoot
|
|
647
|
+
|| transaction.stagingPath !== offloadStagingPath(transaction.objectPath, transaction.id)) {
|
|
648
|
+
throw new Error("archive record no longer owns the path named by its offload journal; every path and the journal were kept");
|
|
649
|
+
}
|
|
650
|
+
// Resolve only the trusted configured root, then validate each descendant
|
|
651
|
+
// one component at a time. This permits platform aliases such as macOS
|
|
652
|
+
// /var -> /private/var without trusting a symlink inserted inside archives/.
|
|
653
|
+
const canonicalRoot = await realpath(lexicalRoot);
|
|
654
|
+
const canonicalObject = join(canonicalRoot, basename(transaction.objectPath));
|
|
655
|
+
const canonicalStaging = join(canonicalRoot, basename(transaction.stagingPath));
|
|
656
|
+
await validateNoFollowPath(canonicalObject, [canonicalRoot], { leaf: "missing-or-file" });
|
|
657
|
+
await validateNoFollowPath(canonicalStaging, [canonicalRoot], { leaf: "missing-or-file" });
|
|
658
|
+
return record;
|
|
659
|
+
}
|
|
660
|
+
async function rollbackOffloadTransaction(dataDir, transaction) {
|
|
661
|
+
await assertOffloadRecordOwnsPath(dataDir, transaction);
|
|
662
|
+
const source = await localArchiveIdentityOrNull(transaction.objectPath);
|
|
663
|
+
const staged = await localArchiveIdentityOrNull(transaction.stagingPath);
|
|
664
|
+
if (source && staged) {
|
|
665
|
+
if (!sameOffloadFile(source, transaction.original)
|
|
666
|
+
|| !sameOffloadFile(staged, transaction.original)
|
|
667
|
+
|| !sameOffloadFile(source, staged)) {
|
|
668
|
+
throw new Error(`Offload conflict kept both ${transaction.objectPath} and ${transaction.stagingPath}`);
|
|
669
|
+
}
|
|
670
|
+
await removeOffloadPathDurably(transaction.stagingPath);
|
|
671
|
+
}
|
|
672
|
+
else if (!source && staged) {
|
|
673
|
+
if (!sameOffloadFile(staged, transaction.original)) {
|
|
674
|
+
throw new Error(`Offload staging conflict kept ${transaction.stagingPath}`);
|
|
675
|
+
}
|
|
676
|
+
await hardLink(transaction.stagingPath, transaction.objectPath);
|
|
677
|
+
await syncOffloadDirectory(dirname(transaction.objectPath));
|
|
678
|
+
const restored = await localArchiveIdentity(transaction.objectPath);
|
|
679
|
+
if (!sameOffloadFile(restored, transaction.original)) {
|
|
680
|
+
throw new Error(`Offload rollback could not prove the restored inode at ${transaction.objectPath}; both names were kept`);
|
|
681
|
+
}
|
|
682
|
+
await removeOffloadPathDurably(transaction.stagingPath);
|
|
683
|
+
}
|
|
684
|
+
else if (source && !staged) {
|
|
685
|
+
if (!sameOffloadFile(source, transaction.original)) {
|
|
686
|
+
throw new Error(`Offload source conflict kept ${transaction.objectPath} and its transaction journal`);
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
else if (!source && !staged) {
|
|
690
|
+
throw new Error("Offload transaction lost both local names before authorization");
|
|
691
|
+
}
|
|
692
|
+
const surviving = await localArchiveIdentityOrNull(transaction.objectPath);
|
|
693
|
+
if (!surviving || !sameOffloadFile(surviving, transaction.original)) {
|
|
694
|
+
throw new Error(`Offload rollback did not retain the exact source inode at ${transaction.objectPath}; metadata and journal were kept`);
|
|
695
|
+
}
|
|
696
|
+
await clearOwnedOffloadMetadata(dataDir, transaction);
|
|
697
|
+
await removeOffloadJournal(dataDir, transaction.id);
|
|
698
|
+
}
|
|
699
|
+
/**
|
|
700
|
+
* Remove the final Sealkeep name while holding the exact inode open. The
|
|
701
|
+
* post-unlink fstat is the truthful answer to "were blocks actually freed?":
|
|
702
|
+
* any pre-existing or concurrently added hard link makes that answer zero.
|
|
703
|
+
*/
|
|
704
|
+
async function purgeOffloadStaging(transaction, beforeUnlink) {
|
|
705
|
+
const expected = transaction.purgeSnapshot;
|
|
706
|
+
if (!expected)
|
|
707
|
+
throw new Error("offload purge was not authorised with a final inode snapshot");
|
|
708
|
+
const handle = await open(transaction.stagingPath, "r");
|
|
709
|
+
try {
|
|
710
|
+
const beforeInfo = await handle.stat({ bigint: true });
|
|
711
|
+
if (!beforeInfo.isFile() || beforeInfo.size > BigInt(Number.MAX_SAFE_INTEGER)
|
|
712
|
+
|| beforeInfo.blocks * 512n > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
713
|
+
throw new Error("offload staging is not a measurable regular file at the purge boundary");
|
|
714
|
+
}
|
|
715
|
+
const before = {
|
|
716
|
+
bytes: Number(beforeInfo.size), allocatedBytes: Number(beforeInfo.blocks * 512n), links: Number(beforeInfo.nlink),
|
|
717
|
+
dev: beforeInfo.dev.toString(), ino: beforeInfo.ino.toString(),
|
|
718
|
+
};
|
|
719
|
+
const named = await localArchiveIdentity(transaction.stagingPath);
|
|
720
|
+
if (!sameOffloadSnapshot(before, expected) || !sameOffloadSnapshot(named, expected)) {
|
|
721
|
+
throw new Error("offload staging changed after purge authorization; it was retained");
|
|
722
|
+
}
|
|
723
|
+
await beforeUnlink?.(transaction);
|
|
724
|
+
await unlink(transaction.stagingPath);
|
|
725
|
+
await syncOffloadDirectory(dirname(transaction.stagingPath));
|
|
726
|
+
const afterInfo = await handle.stat({ bigint: true });
|
|
727
|
+
if (afterInfo.dev.toString() !== expected.dev || afterInfo.ino.toString() !== expected.ino
|
|
728
|
+
|| afterInfo.size !== BigInt(expected.bytes)) {
|
|
729
|
+
throw new Error("offload lost identity of the staged inode while removing its name");
|
|
730
|
+
}
|
|
731
|
+
const remainingLinks = Number(afterInfo.nlink);
|
|
732
|
+
return remainingLinks === 0 ? Number(afterInfo.blocks * 512n) : 0;
|
|
733
|
+
}
|
|
734
|
+
finally {
|
|
735
|
+
await handle.close();
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
async function completeOffloadTransaction(dataDir, transaction, options) {
|
|
739
|
+
let current = transaction;
|
|
740
|
+
let record = await assertOffloadRecordOwnsPath(dataDir, current);
|
|
741
|
+
let source = await localArchiveIdentityOrNull(current.objectPath);
|
|
742
|
+
let staged = await localArchiveIdentityOrNull(current.stagingPath);
|
|
743
|
+
if (!staged && source && current.stage === "prepared") {
|
|
744
|
+
current = await linkOffloadStaging(current);
|
|
745
|
+
await options.operations?.afterLinked?.(current);
|
|
746
|
+
source = await localArchiveIdentityOrNull(current.objectPath);
|
|
747
|
+
staged = await localArchiveIdentityOrNull(current.stagingPath);
|
|
748
|
+
}
|
|
749
|
+
if (!staged || (source && !sameOffloadFile(source, staged)) || !sameOffloadFile(staged, current.original)) {
|
|
750
|
+
throw new Error("offload staging no longer identifies the exact local archive; every surviving path was kept");
|
|
751
|
+
}
|
|
752
|
+
const refreshedRecord = (await listArchives(dataDir)).find((item) => item.id === current.archiveId && isV2(item));
|
|
753
|
+
if (!refreshedRecord)
|
|
754
|
+
throw new Error("archive record disappeared while offload was proving its remote copy");
|
|
755
|
+
record = refreshedRecord;
|
|
756
|
+
if (!offloadMetadataMatches(record, current))
|
|
757
|
+
current = await markOffloadMetadata(dataDir, current);
|
|
758
|
+
else if (current.stage !== "metadata-committed" && current.stage !== "source-unlinked" && current.stage !== "purge-authorized") {
|
|
759
|
+
current = await replaceOffloadJournal(current, "metadata-committed");
|
|
760
|
+
}
|
|
761
|
+
await options.operations?.afterMetadata?.(current);
|
|
762
|
+
source = await localArchiveIdentityOrNull(current.objectPath);
|
|
763
|
+
staged = await localArchiveIdentityOrNull(current.stagingPath);
|
|
764
|
+
if (source) {
|
|
765
|
+
if (!staged || !sameOffloadFile(source, staged) || !sameOffloadFile(source, current.original))
|
|
766
|
+
throw new Error("local archive path was replaced during offload; both files were kept");
|
|
767
|
+
current = await unlinkOffloadSource(current);
|
|
768
|
+
}
|
|
769
|
+
else if (current.stage !== "source-unlinked" && current.stage !== "purge-authorized") {
|
|
770
|
+
current = await replaceOffloadJournal(current, "source-unlinked");
|
|
771
|
+
}
|
|
772
|
+
await options.operations?.afterSourceUnlinked?.(current);
|
|
773
|
+
// The irreversible boundary is removal of the staging name below. Re-read
|
|
774
|
+
// and hash EVERY remote dependency at the frozen common destination here,
|
|
775
|
+
// after all potentially slow local metadata work. In an uninterrupted
|
|
776
|
+
// attempt this is exactly one full streamed read per chain link.
|
|
777
|
+
await verifyOffloadRemoteChain(dataDir, record, current, options);
|
|
778
|
+
current = await replaceOffloadJournal(current, "remote-verified");
|
|
779
|
+
await options.operations?.afterRemoteVerified?.(current);
|
|
780
|
+
const finalStaged = await localArchiveIdentity(current.stagingPath);
|
|
781
|
+
if (!sameOffloadFile(finalStaged, current.original)) {
|
|
782
|
+
throw new Error("offload staging changed at the final purge boundary; it was retained");
|
|
783
|
+
}
|
|
784
|
+
await recordAudit(dataDir, "archive.offload", "allowed", {
|
|
785
|
+
archiveId: current.archiveId,
|
|
786
|
+
transactionId: current.id,
|
|
787
|
+
bytes: current.original.bytes,
|
|
788
|
+
authorizedPhysicalBytes: finalStaged.links === 1 ? finalStaged.allocatedBytes : 0,
|
|
789
|
+
finalLinkCount: finalStaged.links,
|
|
790
|
+
bucket: current.remote.bucket,
|
|
791
|
+
});
|
|
792
|
+
current = await replaceOffloadJournal(current, "purge-authorized", { purgeSnapshot: finalStaged });
|
|
793
|
+
await options.operations?.afterPurgeAuthorized?.(current);
|
|
794
|
+
const freedBytes = await purgeOffloadStaging(current, options.operations?.beforeStagingUnlink);
|
|
795
|
+
await options.operations?.afterStagingPurged?.(current);
|
|
796
|
+
await finalizeOffloadMetadata(dataDir, current, freedBytes);
|
|
797
|
+
const candidate = {
|
|
798
|
+
id: current.archiveId,
|
|
799
|
+
agent: record.source.agent,
|
|
800
|
+
path: record.source.path,
|
|
801
|
+
localBytes: current.original.bytes,
|
|
802
|
+
allocatedBytes: freedBytes,
|
|
803
|
+
provider: current.remote.provider,
|
|
804
|
+
bucket: current.remote.bucket,
|
|
805
|
+
objectKey: current.remote.objectKey,
|
|
806
|
+
...(current.remote.copyId ? { copyId: current.remote.copyId } : {}),
|
|
807
|
+
};
|
|
808
|
+
return { candidate, freedBytes };
|
|
809
|
+
}
|
|
810
|
+
export async function reconcileOffloadTransactions(dataDir, options = {}) {
|
|
811
|
+
const completed = [];
|
|
812
|
+
const failed = [];
|
|
813
|
+
let freedBytes = 0;
|
|
814
|
+
for (let transaction of await readOffloadJournals(dataDir)) {
|
|
815
|
+
const lockKey = createHash("sha256").update(transaction.archiveId).digest("hex");
|
|
816
|
+
const lock = await acquireDurableTicketLock(await trustedOffloadLockPath(dataDir, lockKey), `archive offload ${transaction.archiveId}`, { timeoutMs: 30_000, signal: options.signal });
|
|
817
|
+
try {
|
|
818
|
+
const record = (await listArchives(dataDir)).find((item) => item.id === transaction.archiveId && isV2(item));
|
|
819
|
+
await assertOffloadRecordOwnsPath(dataDir, transaction);
|
|
820
|
+
const source = await localArchiveIdentityOrNull(transaction.objectPath);
|
|
821
|
+
const staged = await localArchiveIdentityOrNull(transaction.stagingPath);
|
|
822
|
+
const ownsMetadata = Boolean(record && offloadMetadataMatches(record, transaction));
|
|
823
|
+
// The record update can reach disk before journal cleanup. That is a
|
|
824
|
+
// completed transaction, not a rollback candidate; both local names
|
|
825
|
+
// must already be gone before this stale journal is retired.
|
|
826
|
+
if (record && offloadMetadataFinalized(record, transaction)) {
|
|
827
|
+
if (source || staged)
|
|
828
|
+
throw new Error("finalized offload metadata conflicts with a surviving local transaction path");
|
|
829
|
+
const released = record.offloaded?.freedBytes ?? 0;
|
|
830
|
+
await removeOffloadJournal(dataDir, transaction.id);
|
|
831
|
+
freedBytes += released;
|
|
832
|
+
completed.push({
|
|
833
|
+
id: transaction.archiveId, agent: record.source.agent, path: record.source.path,
|
|
834
|
+
localBytes: transaction.original.bytes, allocatedBytes: released,
|
|
835
|
+
provider: transaction.remote.provider, bucket: transaction.remote.bucket, objectKey: transaction.remote.objectKey,
|
|
836
|
+
...(transaction.remote.copyId ? { copyId: transaction.remote.copyId } : {}),
|
|
837
|
+
});
|
|
838
|
+
continue;
|
|
839
|
+
}
|
|
840
|
+
if (!ownsMetadata) {
|
|
841
|
+
await rollbackOffloadTransaction(dataDir, transaction);
|
|
842
|
+
continue;
|
|
843
|
+
}
|
|
844
|
+
if (!source && !staged) {
|
|
845
|
+
if (transaction.stage !== "purge-authorized") {
|
|
846
|
+
throw new Error("both local names disappeared before a durable purge authorization; transaction retained for repair");
|
|
847
|
+
}
|
|
848
|
+
// The process died before its post-unlink fstat result reached durable
|
|
849
|
+
// metadata. A hard link may have appeared in the syscall-sized gap,
|
|
850
|
+
// so the only honest recovery metric is the conservative lower bound.
|
|
851
|
+
const released = 0;
|
|
852
|
+
await finalizeOffloadMetadata(dataDir, transaction, released);
|
|
853
|
+
freedBytes += released;
|
|
854
|
+
completed.push({
|
|
855
|
+
id: transaction.archiveId, agent: record.source.agent, path: record.source.path,
|
|
856
|
+
localBytes: transaction.original.bytes, allocatedBytes: released,
|
|
857
|
+
provider: transaction.remote.provider, bucket: transaction.remote.bucket, objectKey: transaction.remote.objectKey,
|
|
858
|
+
...(transaction.remote.copyId ? { copyId: transaction.remote.copyId } : {}),
|
|
859
|
+
});
|
|
860
|
+
continue;
|
|
861
|
+
}
|
|
862
|
+
const outcome = await completeOffloadTransaction(dataDir, transaction, options);
|
|
863
|
+
completed.push(outcome.candidate);
|
|
864
|
+
freedBytes += outcome.freedBytes;
|
|
865
|
+
}
|
|
866
|
+
catch (error) {
|
|
867
|
+
if (options.signal?.aborted)
|
|
868
|
+
throw options.signal.reason ?? error;
|
|
869
|
+
try {
|
|
870
|
+
await rollbackOffloadTransaction(dataDir, transaction);
|
|
871
|
+
}
|
|
872
|
+
catch { /* Every surviving name and the journal remain for a later repair. */ }
|
|
873
|
+
failed.push({ id: transaction.archiveId, error: error instanceof Error ? error.message : "could not reconcile offload" });
|
|
874
|
+
}
|
|
875
|
+
finally {
|
|
876
|
+
await lock.release().catch(() => undefined);
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
return { completed, failed, freedBytes };
|
|
61
880
|
}
|
|
62
881
|
/**
|
|
63
882
|
* Removes local archives that a bucket has a verified copy of.
|
|
@@ -67,51 +886,50 @@ export async function offloadable(dataDir) {
|
|
|
67
886
|
* the deletion of something that has since become the only copy.
|
|
68
887
|
*/
|
|
69
888
|
export async function offloadArchives(dataDir, options = {}) {
|
|
70
|
-
const
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
const
|
|
889
|
+
const recovered = options.confirm
|
|
890
|
+
? await reconcileOffloadTransactions(dataDir, options)
|
|
891
|
+
: { completed: [], failed: [], freedBytes: 0 };
|
|
892
|
+
const all = await offloadable(dataDir, options);
|
|
893
|
+
const recoveryFailures = new Set(recovered.failed.map((item) => item.id));
|
|
894
|
+
const candidates = all.candidates.filter((candidate) => !recoveryFailures.has(candidate.id));
|
|
74
895
|
const blocked = all.blocked;
|
|
75
|
-
const freeableBytes = candidates.reduce((total, item) => total + item.
|
|
896
|
+
const freeableBytes = candidates.reduce((total, item) => total + item.allocatedBytes, 0);
|
|
76
897
|
if (!options.confirm)
|
|
77
898
|
return { mode: "preview", wouldOffload: candidates, blocked, freedBytes: freeableBytes };
|
|
78
|
-
const
|
|
79
|
-
const
|
|
80
|
-
|
|
81
|
-
const config = await readConfig(dataDir);
|
|
899
|
+
const offloaded = [...recovered.completed];
|
|
900
|
+
const failed = [...recovered.failed];
|
|
901
|
+
let freedBytes = recovered.freedBytes;
|
|
82
902
|
for (const candidate of candidates) {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
903
|
+
options.signal?.throwIfAborted();
|
|
904
|
+
const lockKey = createHash("sha256").update(candidate.id).digest("hex");
|
|
905
|
+
const lock = await acquireDurableTicketLock(await trustedOffloadLockPath(dataDir, lockKey), `archive offload ${candidate.id}`, { timeoutMs: 30_000, signal: options.signal });
|
|
906
|
+
let transaction = null;
|
|
86
907
|
try {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
if (sha256(fetched) !== record.cipher.ciphertextSha256) {
|
|
96
|
-
failed.push({ id: candidate.id, error: "what the bucket returned is not this archive; local copy kept" });
|
|
97
|
-
continue;
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
await rm(record.objectPath, { force: true });
|
|
101
|
-
const next = {
|
|
102
|
-
...record,
|
|
103
|
-
offloaded: { at: new Date().toISOString(), provider: candidate.provider, bucket: candidate.bucket, objectKey: candidate.objectKey }
|
|
104
|
-
};
|
|
105
|
-
await writeRecord(config.storage.root, next);
|
|
106
|
-
await recordAudit(dataDir, "archive.offload", "allowed", { archiveId: candidate.id, bytes: candidate.localBytes, bucket: candidate.bucket });
|
|
107
|
-
offloaded.push(candidate);
|
|
908
|
+
const record = (await listArchives(dataDir)).find((item) => item.id === candidate.id && isV2(item));
|
|
909
|
+
if (!record || record.offloaded)
|
|
910
|
+
continue;
|
|
911
|
+
transaction = await beginOffloadTransaction(dataDir, candidate, record);
|
|
912
|
+
await options.operations?.afterJournal?.(transaction);
|
|
913
|
+
const outcome = await completeOffloadTransaction(dataDir, transaction, options);
|
|
914
|
+
offloaded.push(outcome.candidate);
|
|
915
|
+
freedBytes += outcome.freedBytes;
|
|
108
916
|
}
|
|
109
917
|
catch (error) {
|
|
110
|
-
|
|
918
|
+
if (options.signal?.aborted)
|
|
919
|
+
throw options.signal.reason ?? error;
|
|
920
|
+
if (transaction)
|
|
921
|
+
await rollbackOffloadTransaction(dataDir, transaction).catch(() => undefined);
|
|
922
|
+
await recordAudit(dataDir, "archive.offload", "denied", {
|
|
923
|
+
archiveId: candidate.id,
|
|
924
|
+
...(transaction ? { transactionId: transaction.id } : {}),
|
|
925
|
+
});
|
|
111
926
|
failed.push({ id: candidate.id, error: error instanceof Error ? error.message : "could not offload" });
|
|
112
927
|
}
|
|
928
|
+
finally {
|
|
929
|
+
await lock.release().catch(() => undefined);
|
|
930
|
+
}
|
|
113
931
|
}
|
|
114
|
-
return { mode: "apply", offloaded, failed, blocked, freedBytes
|
|
932
|
+
return { mode: "apply", offloaded, failed, blocked, freedBytes };
|
|
115
933
|
}
|
|
116
934
|
/**
|
|
117
935
|
* Downloads an archive's ciphertext from the bucket it was offloaded to.
|
|
@@ -122,103 +940,115 @@ export async function offloadArchives(dataDir, options = {}) {
|
|
|
122
940
|
* not be recovered by the person who owns it — which would turn a space-saving
|
|
123
941
|
* feature into data loss. The flag is set for this call only.
|
|
124
942
|
*/
|
|
125
|
-
async function
|
|
126
|
-
|
|
127
|
-
|
|
943
|
+
export async function fetchCiphertextStream(dataDir, record, client, selectedCopy, overrides = {}, signal, idleMs, maxBytesPerSecond) {
|
|
944
|
+
return byteStreamWithIdleTimeout(async (readSignal) => {
|
|
945
|
+
const remote = selectedCopy ?? archiveCopies(record)[0] ?? record.offloaded;
|
|
946
|
+
const managedFrame = remote?.provider === "vaultline" && !client && remote.layout?.kind !== "chunks";
|
|
947
|
+
const stream = await openCiphertextStream(dataDir, record, client, selectedCopy, overrides, readSignal, maxBytesPerSecond);
|
|
948
|
+
// Managed frames are paced before their metadata is stripped. Other
|
|
949
|
+
// providers expose raw ciphertext, which is paced exactly once here.
|
|
950
|
+
return maxBytesPerSecond === undefined || managedFrame ? stream
|
|
951
|
+
: paceBackgroundByteStream(dataDir, stream, maxBytesPerSecond, { signal: readSignal });
|
|
952
|
+
}, { signal, idleMs });
|
|
953
|
+
}
|
|
954
|
+
async function openCiphertextStream(dataDir, record, client, selectedCopy, overrides, signal, maxBytesPerSecond) {
|
|
955
|
+
const remote = selectedCopy ?? archiveCopies(record)[0] ?? record.offloaded;
|
|
128
956
|
if (!remote)
|
|
129
957
|
fail("archive_not_found", `Archive ${record.id} has no remote copy to fetch`, { archiveId: record.id });
|
|
130
958
|
// A managed copy reads back through the account, not a provider credential:
|
|
131
959
|
// pullCiphertext hash-checks what it downloads before returning it. Branch
|
|
132
960
|
// on the record's own provider — a vault can hold managed history alongside
|
|
133
961
|
// a later own-bucket configuration, and each copy fetches its own way.
|
|
134
|
-
if (remote.provider === "vaultline" && !client &&
|
|
135
|
-
const {
|
|
136
|
-
const { ciphertext: blob } = await pullCiphertext(dataDir, record.id);
|
|
962
|
+
if (remote.provider === "vaultline" && !client && remote.layout?.kind !== "chunks") {
|
|
963
|
+
const { pullCiphertextStream } = await import("./cloud.js");
|
|
137
964
|
// A managed object is envelope-framed — the self-carrying wrapper push
|
|
138
965
|
// added so a bare object can be opened anywhere. What the local record
|
|
139
966
|
// hashes is the ciphertext INSIDE the frame, so that is what this must
|
|
140
967
|
// return; handing back the framed bytes made every integrity check
|
|
141
968
|
// compare an apple to the crate it shipped in. A plain-mode object has
|
|
142
969
|
// no frame and IS the stored bytes.
|
|
143
|
-
|
|
144
|
-
return envelope ? ciphertext : blob;
|
|
970
|
+
return pullCiphertextStream(dataDir, record.id, undefined, { unframe: true, signal, maxBytesPerSecond });
|
|
145
971
|
}
|
|
146
972
|
// Chunk-folder remotes fetch object by object, each hash-checked against
|
|
147
973
|
// the envelope's own chunk headers — the layout was made for exactly this.
|
|
148
974
|
// A managed folder reads back through the account's download leases; an own
|
|
149
975
|
// bucket reads through the provider's object verbs.
|
|
150
|
-
if (
|
|
151
|
-
const {
|
|
976
|
+
if (remote.layout?.kind === "chunks") {
|
|
977
|
+
const { fetchChunkFolderStream } = await import("./chunk-store.js");
|
|
152
978
|
let folderClient;
|
|
153
979
|
if (client)
|
|
154
980
|
folderClient = client;
|
|
155
|
-
else if (
|
|
981
|
+
else if (remote.provider === "vaultline") {
|
|
156
982
|
folderClient = (await import("./managed-chunks.js")).managedChunkClient(dataDir);
|
|
157
983
|
}
|
|
158
984
|
else {
|
|
159
|
-
const
|
|
160
|
-
|
|
985
|
+
const named = await configuredTargetForRemote(dataDir, remote);
|
|
986
|
+
if (named.provider === "vaultline" || !named.bucket)
|
|
987
|
+
fail("storage_not_configured", "This chunk copy has no usable bucket target on this machine");
|
|
988
|
+
const target = { provider: named.provider, bucket: named.bucket, prefix: named.prefix ?? "sealkeep", ...(named.region ? { region: named.region } : {}) };
|
|
989
|
+
const connection = await ownBucketConnection(dataDir, named, overrides);
|
|
990
|
+
folderClient = await uploadClientFromStore(dataDir, connection.credentialId, target, connection.overrides, undefined);
|
|
161
991
|
}
|
|
162
992
|
if (typeof folderClient.getObject !== "function")
|
|
163
|
-
fail("provider_unsupported", `${
|
|
164
|
-
return
|
|
993
|
+
fail("provider_unsupported", `${remote.provider} cannot fetch chunk folders with this build`);
|
|
994
|
+
return fetchChunkFolderStream(folderClient, remote, record.envelope, { signal });
|
|
165
995
|
}
|
|
166
996
|
// The record routes its own read: provider and bucket come from the copy
|
|
167
997
|
// itself, the prefix from its object key, and a configured target (matched
|
|
168
998
|
// by targetId, then by provider+bucket) fills in the rest. config.remoteStorage
|
|
169
999
|
// is only the last resort — a multi-target vault has none.
|
|
170
|
-
const
|
|
171
|
-
|
|
172
|
-
return config.remoteStorage;
|
|
173
|
-
try {
|
|
174
|
-
const { resolveTargets } = await import("./storage-targets.js");
|
|
175
|
-
const targets = await resolveTargets(dataDir);
|
|
176
|
-
const stampedId = remote.targetId;
|
|
177
|
-
const match = (stampedId ? targets.find((candidate) => candidate.id === stampedId) : undefined)
|
|
178
|
-
?? targets.find((candidate) => candidate.provider === remote.provider && candidate.bucket === remote.bucket);
|
|
179
|
-
if (match) {
|
|
180
|
-
const derivedPrefix = remote.objectKey.split("/").slice(0, -1).join("/");
|
|
181
|
-
return { provider: match.provider, bucket: match.bucket ?? remote.bucket, prefix: match.prefix ?? (derivedPrefix || "vaultline"), region: match.region };
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
catch { /* fall through */ }
|
|
185
|
-
if (config.remoteStorage)
|
|
186
|
-
return config.remoteStorage;
|
|
187
|
-
return fail("storage_not_configured", "This vault has no storage configured, so an offloaded archive cannot be fetched back");
|
|
188
|
-
})();
|
|
189
|
-
if (target.provider === "vaultline")
|
|
1000
|
+
const named = await configuredTargetForRemote(dataDir, remote);
|
|
1001
|
+
if (named.provider === "vaultline" || !named.bucket)
|
|
190
1002
|
fail("storage_not_configured", "This managed copy should have taken the managed read path");
|
|
191
|
-
const
|
|
192
|
-
const
|
|
193
|
-
|
|
194
|
-
|
|
1003
|
+
const derivedPrefix = remote.objectKey.split("/").slice(0, -1).join("/");
|
|
1004
|
+
const target = {
|
|
1005
|
+
provider: named.provider,
|
|
1006
|
+
bucket: named.bucket,
|
|
1007
|
+
prefix: named.prefix ?? (derivedPrefix || "sealkeep"),
|
|
1008
|
+
...(named.region ? { region: named.region } : {}),
|
|
1009
|
+
};
|
|
1010
|
+
const connection = client ? null : await ownBucketConnection(dataDir, named, overrides);
|
|
1011
|
+
const resolved = client ?? await uploadClientFromStore(dataDir, connection.credentialId, target, connection.overrides, undefined);
|
|
1012
|
+
const lease = { ...createActiveLease(target, {
|
|
1013
|
+
archiveId: record.id, ciphertextSha256: record.cipher.ciphertextSha256, bytes: record.cipher.storedBytes
|
|
1014
|
+
}, { ...process.env, SEALKEEP_ENABLE_SIGNER: "1" }), objectKey: remote.objectKey };
|
|
195
1015
|
const readable = resolved;
|
|
1016
|
+
if (typeof readable.downloadStream === "function")
|
|
1017
|
+
return readable.downloadStream(lease, { signal });
|
|
196
1018
|
if (typeof readable.download !== "function") {
|
|
197
1019
|
fail("provider_unsupported", `${target.provider} cannot be read back by this build, so this archive cannot be fetched`);
|
|
198
1020
|
}
|
|
199
|
-
|
|
1021
|
+
// Small injected/test clients predate the stream contract. Never invoke a
|
|
1022
|
+
// whole-Buffer client for a large archive: fail before it can allocate.
|
|
1023
|
+
if (record.cipher.storedBytes > LEGACY_BUFFER_LIMIT_BYTES) {
|
|
1024
|
+
fail("provider_unsupported", `${target.provider} cannot stream this ${record.cipher.storedBytes}-byte archive with the installed client`, { archiveId: record.id });
|
|
1025
|
+
}
|
|
1026
|
+
const buffered = await readable.download(lease, { signal });
|
|
1027
|
+
if (buffered.length > LEGACY_BUFFER_LIMIT_BYTES) {
|
|
1028
|
+
fail("provider_unsupported", `${target.provider} returned more than the bounded compatibility limit`, { archiveId: record.id });
|
|
1029
|
+
}
|
|
1030
|
+
return oneChunk(buffered);
|
|
200
1031
|
}
|
|
201
1032
|
/**
|
|
202
1033
|
* One managed record checked against the account's rows. A single-object copy
|
|
203
1034
|
* is one row at the recorded size; a chunk-folder copy is its whole family —
|
|
204
1035
|
* every chunk row present, the envelope sidecar present, and the chunk bytes
|
|
205
1036
|
* summing to the verified total. The proof standard does not soften because
|
|
206
|
-
* the object count went up: any missing member is "do not
|
|
1037
|
+
* the object count went up: any missing member is "do not reclaim the original".
|
|
207
1038
|
*/
|
|
208
|
-
async function managedCopyCheck(dataDir, record, rows, checkedAt) {
|
|
209
|
-
const remote = record.remote;
|
|
1039
|
+
async function managedCopyCheck(dataDir, record, remote, rows, checkedAt) {
|
|
210
1040
|
if (remote.layout?.kind === "chunks") {
|
|
211
1041
|
const { managedChunkRefs } = await import("./managed-chunks.js");
|
|
212
1042
|
const refs = managedChunkRefs(record.id, remote.layout.chunkCount);
|
|
213
1043
|
const missing = refs.chunks.filter((ref) => !rows.has(ref)).length + (rows.has(refs.envelope) ? 0 : 1);
|
|
214
1044
|
if (missing > 0) {
|
|
215
1045
|
await recordAudit(dataDir, "remote.check", "denied", { archiveId: record.id, reason: "absent", missing });
|
|
216
|
-
return { ok: false, provider: remote.provider, bytes: 0, checkedAt, detail: `Your managed storage is missing ${missing} of this archive's ${refs.chunks.length + 1} objects. Do not
|
|
1046
|
+
return { ok: false, provider: remote.provider, bytes: 0, checkedAt, detail: `Your managed storage is missing ${missing} of this archive's ${refs.chunks.length + 1} objects. Do not reclaim the original — this disk may hold the only complete copy.` };
|
|
217
1047
|
}
|
|
218
1048
|
const bytes = refs.chunks.reduce((sum, ref) => sum + (rows.get(ref) ?? 0), 0);
|
|
219
1049
|
if (bytes !== remote.bytes) {
|
|
220
1050
|
await recordAudit(dataDir, "remote.check", "denied", { archiveId: record.id, reason: "size", stored: bytes });
|
|
221
|
-
return { ok: false, provider: remote.provider, bytes, checkedAt, detail: `Your storage reports ${bytes} chunk bytes; ${remote.bytes} were verified. Do not
|
|
1051
|
+
return { ok: false, provider: remote.provider, bytes, checkedAt, detail: `Your storage reports ${bytes} chunk bytes; ${remote.bytes} were verified. Do not reclaim the original.` };
|
|
222
1052
|
}
|
|
223
1053
|
await recordAudit(dataDir, "remote.check", "allowed", { archiveId: record.id, bytes, chunks: refs.chunks.length });
|
|
224
1054
|
return { ok: true, provider: remote.provider, bytes, checkedAt, detail: `Your managed storage confirmed all ${refs.chunks.length} chunks and the sidecar at the recorded size, just now.` };
|
|
@@ -226,49 +1056,98 @@ async function managedCopyCheck(dataDir, record, rows, checkedAt) {
|
|
|
226
1056
|
const stored = rows.get(record.id);
|
|
227
1057
|
if (stored === undefined) {
|
|
228
1058
|
await recordAudit(dataDir, "remote.check", "denied", { archiveId: record.id, reason: "absent" });
|
|
229
|
-
return { ok: false, provider: remote.provider, bytes: 0, checkedAt, detail: "Your managed storage no longer lists this archive. Do not
|
|
1059
|
+
return { ok: false, provider: remote.provider, bytes: 0, checkedAt, detail: "Your managed storage no longer lists this archive. Do not reclaim the original — this disk may hold the only copy." };
|
|
230
1060
|
}
|
|
231
1061
|
if (stored !== remote.bytes) {
|
|
232
1062
|
await recordAudit(dataDir, "remote.check", "denied", { archiveId: record.id, reason: "size", stored });
|
|
233
|
-
return { ok: false, provider: remote.provider, bytes: stored, checkedAt, detail: `Your storage reports ${stored} bytes; ${remote.bytes} were verified. Do not
|
|
1063
|
+
return { ok: false, provider: remote.provider, bytes: stored, checkedAt, detail: `Your storage reports ${stored} bytes; ${remote.bytes} were verified. Do not reclaim the original.` };
|
|
234
1064
|
}
|
|
235
1065
|
await recordAudit(dataDir, "remote.check", "allowed", { archiveId: record.id, bytes: stored });
|
|
236
1066
|
return { ok: true, provider: remote.provider, bytes: stored, checkedAt, detail: "Your managed storage confirmed it holds this archive at the recorded size, just now." };
|
|
237
1067
|
}
|
|
238
|
-
export async function checkRemoteCopy(dataDir, archiveId, client) {
|
|
1068
|
+
export async function checkRemoteCopy(dataDir, archiveId, client, copyId) {
|
|
239
1069
|
const record = (await listArchives(dataDir)).find((item) => item.id === archiveId);
|
|
240
1070
|
if (!record)
|
|
241
1071
|
fail("archive_not_found", `Archive not found: ${archiveId}`, { archiveId });
|
|
242
1072
|
if (!isV2(record))
|
|
243
1073
|
fail("invalid_argument", `Archive ${archiveId} uses format v1. Run: sealkeep migrate`, { archiveId });
|
|
244
|
-
const remote =
|
|
1074
|
+
const remote = copyId
|
|
1075
|
+
? archiveCopies(record).find((copy) => copy.copyId === copyId)
|
|
1076
|
+
: archiveCopies(record).find((copy) => copy.verifiedAt);
|
|
245
1077
|
if (!remote?.verifiedAt)
|
|
246
1078
|
fail("invalid_argument", "This archive has no verified cloud copy to check — upload it first.", { archiveId });
|
|
247
1079
|
const checkedAt = new Date().toISOString();
|
|
248
1080
|
if (remote.provider === "vaultline" && !client) {
|
|
249
1081
|
const { listCloudArchives } = await import("./cloud.js");
|
|
250
1082
|
const rows = new Map((await listCloudArchives(dataDir)).map((row) => [row.vault_ref, Number(row.bytes) || 0]));
|
|
251
|
-
return managedCopyCheck(dataDir, record, rows, checkedAt);
|
|
1083
|
+
return managedCopyCheck(dataDir, record, remote, rows, checkedAt);
|
|
252
1084
|
}
|
|
253
|
-
const
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
const
|
|
258
|
-
const
|
|
259
|
-
|
|
260
|
-
|
|
1085
|
+
const named = await configuredTargetForRemote(dataDir, remote);
|
|
1086
|
+
if (named.provider === "vaultline" || !named.bucket) {
|
|
1087
|
+
fail("storage_not_configured", `The ${remote.provider} destination holding this copy is not connected on this machine, so it cannot be checked.`);
|
|
1088
|
+
}
|
|
1089
|
+
const derivedPrefix = remote.objectKey.split("/").slice(0, -1).join("/");
|
|
1090
|
+
const target = {
|
|
1091
|
+
provider: named.provider,
|
|
1092
|
+
bucket: named.bucket,
|
|
1093
|
+
prefix: named.prefix ?? (derivedPrefix || "sealkeep"),
|
|
1094
|
+
...(named.region ? { region: named.region } : {}),
|
|
1095
|
+
};
|
|
1096
|
+
const connection = client ? null : await ownBucketConnection(dataDir, named);
|
|
1097
|
+
const resolved = client ?? await uploadClientFromStore(dataDir, connection.credentialId, target, connection.overrides, undefined);
|
|
1098
|
+
if (remote.layout?.kind === "chunks") {
|
|
1099
|
+
const probing = resolved;
|
|
1100
|
+
if (typeof probing.headObject !== "function")
|
|
1101
|
+
fail("provider_unsupported", `${remote.provider} cannot probe chunk folders with this build`);
|
|
1102
|
+
if (remote.layout.chunkCount !== record.envelope.chunks.length) {
|
|
1103
|
+
return { ok: false, provider: remote.provider, bytes: 0, checkedAt, detail: `The record names ${record.envelope.chunks.length} chunks but this copy claims ${remote.layout.chunkCount}. Do not reclaim the original.` };
|
|
1104
|
+
}
|
|
1105
|
+
let bytes = 0;
|
|
1106
|
+
const problems = [];
|
|
1107
|
+
const { chunkObjectName, ENVELOPE_OBJECT } = await import("./chunk-store.js");
|
|
1108
|
+
for (let index = 0; index < record.envelope.chunks.length; index += 1) {
|
|
1109
|
+
const expected = record.envelope.chunks[index].storedBytes;
|
|
1110
|
+
const head = await probing.headObject(`${remote.objectKey}/${chunkObjectName(index)}`);
|
|
1111
|
+
if (!head.exists)
|
|
1112
|
+
problems.push(`chunk ${index} is missing`);
|
|
1113
|
+
else {
|
|
1114
|
+
bytes += head.bytes;
|
|
1115
|
+
if (head.bytes !== expected)
|
|
1116
|
+
problems.push(`chunk ${index} is ${head.bytes} bytes; ${expected} were verified`);
|
|
1117
|
+
}
|
|
1118
|
+
if (problems.length >= 3)
|
|
1119
|
+
break;
|
|
1120
|
+
}
|
|
1121
|
+
if (problems.length < 3) {
|
|
1122
|
+
const sidecar = await probing.headObject(`${remote.objectKey}/${ENVELOPE_OBJECT}`);
|
|
1123
|
+
if (!sidecar.exists)
|
|
1124
|
+
problems.push("the envelope sidecar is missing");
|
|
1125
|
+
else if (sidecar.bytes <= 0)
|
|
1126
|
+
problems.push("the envelope sidecar is empty");
|
|
1127
|
+
}
|
|
1128
|
+
if (bytes !== remote.bytes)
|
|
1129
|
+
problems.push(`the chunks total ${bytes} bytes; ${remote.bytes} were verified`);
|
|
1130
|
+
if (problems.length > 0) {
|
|
1131
|
+
await recordAudit(dataDir, "remote.check", "denied", { archiveId, reason: "chunk-folder", problems: problems.length, stored: bytes });
|
|
1132
|
+
return { ok: false, provider: remote.provider, bytes, checkedAt, detail: `${problems.join("; ")}. Do not reclaim the original — this disk may hold the only complete copy.` };
|
|
1133
|
+
}
|
|
1134
|
+
await recordAudit(dataDir, "remote.check", "allowed", { archiveId, bytes, chunks: remote.layout.chunkCount, sidecar: true });
|
|
1135
|
+
return { ok: true, provider: remote.provider, bytes, checkedAt, detail: `The bucket confirmed all ${remote.layout.chunkCount} chunks and the envelope sidecar at the recorded sizes, just now.` };
|
|
1136
|
+
}
|
|
1137
|
+
const lease = { ...createActiveLease(target, {
|
|
1138
|
+
archiveId: record.id, ciphertextSha256: record.cipher.ciphertextSha256, bytes: record.cipher.storedBytes
|
|
1139
|
+
}, { ...process.env, SEALKEEP_ENABLE_SIGNER: "1" }), objectKey: remote.objectKey };
|
|
261
1140
|
const probing = resolved;
|
|
262
1141
|
if (typeof probing.head !== "function")
|
|
263
|
-
fail("provider_unsupported", `${
|
|
1142
|
+
fail("provider_unsupported", `${remote.provider} cannot be probed by this build`);
|
|
264
1143
|
const head = await probing.head(lease);
|
|
265
1144
|
if (!head.exists) {
|
|
266
1145
|
await recordAudit(dataDir, "remote.check", "denied", { archiveId, reason: "absent" });
|
|
267
|
-
return { ok: false, provider: remote.provider, bytes: 0, checkedAt, detail: "The bucket no longer holds this object. Do not
|
|
1146
|
+
return { ok: false, provider: remote.provider, bytes: 0, checkedAt, detail: "The bucket no longer holds this object. Do not reclaim the original — this disk may hold the only copy." };
|
|
268
1147
|
}
|
|
269
1148
|
if (head.bytes !== remote.bytes) {
|
|
270
1149
|
await recordAudit(dataDir, "remote.check", "denied", { archiveId, reason: "size", stored: head.bytes });
|
|
271
|
-
return { ok: false, provider: remote.provider, bytes: head.bytes, checkedAt, detail: `The bucket holds ${head.bytes} bytes; ${remote.bytes} were verified. Do not
|
|
1150
|
+
return { ok: false, provider: remote.provider, bytes: head.bytes, checkedAt, detail: `The bucket holds ${head.bytes} bytes; ${remote.bytes} were verified. Do not reclaim the original.` };
|
|
272
1151
|
}
|
|
273
1152
|
await recordAudit(dataDir, "remote.check", "allowed", { archiveId, bytes: head.bytes });
|
|
274
1153
|
return { ok: true, provider: remote.provider, bytes: head.bytes, checkedAt, detail: "The bucket answered just now and holds this archive at the recorded size." };
|
|
@@ -283,7 +1162,7 @@ export async function checkRemoteCopyMany(dataDir, ids, client) {
|
|
|
283
1162
|
const wanted = ids.map((id) => archives.find((item) => item.id === id));
|
|
284
1163
|
const checkedAt = new Date().toISOString();
|
|
285
1164
|
const results = [];
|
|
286
|
-
const managedIds = wanted.filter((record) => !!record && isV2(record) && record.
|
|
1165
|
+
const managedIds = wanted.filter((record) => !!record && isV2(record) && archiveCopies(record).some((copy) => copy.verifiedAt && copy.provider === "vaultline")).map((record) => record.id);
|
|
287
1166
|
let managedRows = null;
|
|
288
1167
|
if (managedIds.length > 0 && !client) {
|
|
289
1168
|
const { listCloudArchives } = await import("./cloud.js");
|
|
@@ -291,22 +1170,718 @@ export async function checkRemoteCopyMany(dataDir, ids, client) {
|
|
|
291
1170
|
}
|
|
292
1171
|
for (const id of ids) {
|
|
293
1172
|
const record = archives.find((item) => item.id === id);
|
|
294
|
-
if (!record || !isV2(record)
|
|
1173
|
+
if (!record || !isV2(record)) {
|
|
295
1174
|
results.push({ ok: false, provider: "none", bytes: 0, checkedAt, detail: "No verified cloud copy on record for this archive." });
|
|
296
1175
|
continue;
|
|
297
1176
|
}
|
|
298
|
-
|
|
299
|
-
|
|
1177
|
+
const copies = archiveCopies(record).filter((copy) => copy.verifiedAt);
|
|
1178
|
+
if (copies.length === 0) {
|
|
1179
|
+
results.push({ ok: false, provider: "none", bytes: 0, checkedAt, detail: "No verified cloud copy on record for this archive." });
|
|
300
1180
|
continue;
|
|
301
1181
|
}
|
|
1182
|
+
const checks = [];
|
|
1183
|
+
for (const copy of copies) {
|
|
1184
|
+
try {
|
|
1185
|
+
if (copy.provider === "vaultline" && managedRows && !client) {
|
|
1186
|
+
checks.push(await managedCopyCheck(dataDir, record, copy, managedRows, checkedAt));
|
|
1187
|
+
}
|
|
1188
|
+
else {
|
|
1189
|
+
checks.push(await checkRemoteCopy(dataDir, id, client, copy.copyId));
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
catch (error) {
|
|
1193
|
+
checks.push({ ok: false, provider: copy.provider, bytes: 0, checkedAt, detail: error instanceof Error ? error.message : "the check itself failed" });
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
const failed = checks.find((check) => !check.ok);
|
|
1197
|
+
results.push(failed ?? {
|
|
1198
|
+
ok: true,
|
|
1199
|
+
provider: checks.length === 1 ? checks[0].provider : "multiple",
|
|
1200
|
+
bytes: checks.reduce((sum, check) => sum + check.bytes, 0),
|
|
1201
|
+
checkedAt,
|
|
1202
|
+
detail: checks.length === 1 ? checks[0].detail : `All ${checks.length} stored copies answered at their recorded sizes, just now.`,
|
|
1203
|
+
});
|
|
1204
|
+
}
|
|
1205
|
+
return results;
|
|
1206
|
+
}
|
|
1207
|
+
function cancellationError(signal) {
|
|
1208
|
+
if (signal?.reason instanceof Error)
|
|
1209
|
+
return signal.reason;
|
|
1210
|
+
const error = new Error("The archive materialisation was cancelled");
|
|
1211
|
+
error.name = "AbortError";
|
|
1212
|
+
return error;
|
|
1213
|
+
}
|
|
1214
|
+
const DEFAULT_SCRATCH_WAIT_MS = 5_000;
|
|
1215
|
+
function scratchBusyError(waitMs) {
|
|
1216
|
+
return new SealkeepError("scratch_space_busy", `Another archive operation is using this disk's temporary workspace. Sealkeep waited ${Math.max(0, Math.round(waitMs / 100) / 10)} seconds without taking extra space; try again after that operation finishes.`, { retryable: true, waitedMs: waitMs });
|
|
1217
|
+
}
|
|
1218
|
+
function waitForScratchLease(signal, delayMs) {
|
|
1219
|
+
if (signal?.aborted)
|
|
1220
|
+
return Promise.reject(cancellationError(signal));
|
|
1221
|
+
return new Promise((resolve, reject) => {
|
|
1222
|
+
const finish = () => {
|
|
1223
|
+
if (signal)
|
|
1224
|
+
signal.removeEventListener("abort", abort);
|
|
1225
|
+
resolve();
|
|
1226
|
+
};
|
|
1227
|
+
const timer = setTimeout(finish, Math.max(1, delayMs));
|
|
1228
|
+
const abort = () => {
|
|
1229
|
+
clearTimeout(timer);
|
|
1230
|
+
signal?.removeEventListener("abort", abort);
|
|
1231
|
+
reject(cancellationError(signal));
|
|
1232
|
+
};
|
|
1233
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
1234
|
+
});
|
|
1235
|
+
}
|
|
1236
|
+
/**
|
|
1237
|
+
* Cleanup is secondary to the operation that made cleanup necessary. Preserve
|
|
1238
|
+
* the original Error object (and therefore a SealkeepError's code/details or
|
|
1239
|
+
* an AbortSignal's exact reason) while making a failed scratch release visible
|
|
1240
|
+
* to logs and callers. Frozen/non-Error throwables fall back to AggregateError,
|
|
1241
|
+
* which still retains both original values in order.
|
|
1242
|
+
*/
|
|
1243
|
+
function withScratchReleaseFailure(primary, releaseError) {
|
|
1244
|
+
const releaseMessage = releaseError instanceof Error ? releaseError.message : "unknown scratch release failure";
|
|
1245
|
+
const detail = `releasing the archive scratch reservation also failed (${releaseMessage})`;
|
|
1246
|
+
if (primary instanceof Error) {
|
|
1247
|
+
try {
|
|
1248
|
+
Object.defineProperty(primary, "scratchReleaseError", {
|
|
1249
|
+
value: releaseError,
|
|
1250
|
+
configurable: true,
|
|
1251
|
+
enumerable: false,
|
|
1252
|
+
});
|
|
1253
|
+
primary.message = `${primary.message}; ${detail}`;
|
|
1254
|
+
return primary;
|
|
1255
|
+
}
|
|
1256
|
+
catch { /* a frozen/custom Error is represented by the aggregate below */ }
|
|
1257
|
+
}
|
|
1258
|
+
const primaryMessage = primary instanceof Error ? primary.message : String(primary);
|
|
1259
|
+
return new AggregateError([primary, releaseError], `${primaryMessage}; ${detail}`, { cause: primary });
|
|
1260
|
+
}
|
|
1261
|
+
/**
|
|
1262
|
+
* One complete fetched archive at a time in this process.
|
|
1263
|
+
*
|
|
1264
|
+
* Historical indexing now consumes chunk-independent archives directly from
|
|
1265
|
+
* their authenticated provider stream and never enters this gate. Operations
|
|
1266
|
+
* that genuinely require a complete file (restore, share, or provider bridge)
|
|
1267
|
+
* may still overlap across local servers. The permit is held until the verified
|
|
1268
|
+
* temporary file is released, not merely until the download ends: measured free
|
|
1269
|
+
* space therefore reflects the previous archive's deletion before the next
|
|
1270
|
+
* check. Local archives never enter this gate and retain their parallel path.
|
|
1271
|
+
*/
|
|
1272
|
+
class ScratchMaterialisationGate {
|
|
1273
|
+
active = false;
|
|
1274
|
+
waiting = [];
|
|
1275
|
+
acquire(signal, deadline, waitMs) {
|
|
1276
|
+
if (signal?.aborted)
|
|
1277
|
+
return Promise.reject(cancellationError(signal));
|
|
1278
|
+
if (Date.now() >= deadline)
|
|
1279
|
+
return Promise.reject(scratchBusyError(waitMs));
|
|
1280
|
+
return new Promise((grant, reject) => {
|
|
1281
|
+
const waiter = { signal, grant, reject };
|
|
1282
|
+
const withdraw = (error) => {
|
|
1283
|
+
const at = this.waiting.indexOf(waiter);
|
|
1284
|
+
if (at >= 0)
|
|
1285
|
+
this.waiting.splice(at, 1);
|
|
1286
|
+
if (waiter.timer)
|
|
1287
|
+
clearTimeout(waiter.timer);
|
|
1288
|
+
signal?.removeEventListener("abort", waiter.onAbort);
|
|
1289
|
+
reject(error);
|
|
1290
|
+
};
|
|
1291
|
+
if (signal) {
|
|
1292
|
+
waiter.onAbort = () => withdraw(cancellationError(signal));
|
|
1293
|
+
signal.addEventListener("abort", waiter.onAbort, { once: true });
|
|
1294
|
+
}
|
|
1295
|
+
waiter.timer = setTimeout(() => withdraw(scratchBusyError(waitMs)), Math.max(1, deadline - Date.now()));
|
|
1296
|
+
if (!this.active)
|
|
1297
|
+
this.activate(waiter);
|
|
1298
|
+
else
|
|
1299
|
+
this.waiting.push(waiter);
|
|
1300
|
+
});
|
|
1301
|
+
}
|
|
1302
|
+
activate(waiter) {
|
|
1303
|
+
if (waiter.signal?.aborted) {
|
|
1304
|
+
waiter.onAbort?.();
|
|
1305
|
+
this.advance();
|
|
1306
|
+
return;
|
|
1307
|
+
}
|
|
1308
|
+
this.active = true;
|
|
1309
|
+
if (waiter.timer)
|
|
1310
|
+
clearTimeout(waiter.timer);
|
|
1311
|
+
if (waiter.signal && waiter.onAbort)
|
|
1312
|
+
waiter.signal.removeEventListener("abort", waiter.onAbort);
|
|
1313
|
+
let released = false;
|
|
1314
|
+
waiter.grant(() => {
|
|
1315
|
+
if (released)
|
|
1316
|
+
return;
|
|
1317
|
+
released = true;
|
|
1318
|
+
this.active = false;
|
|
1319
|
+
this.advance();
|
|
1320
|
+
});
|
|
1321
|
+
}
|
|
1322
|
+
advance() {
|
|
1323
|
+
if (this.active)
|
|
1324
|
+
return;
|
|
1325
|
+
const next = this.waiting.shift();
|
|
1326
|
+
if (next)
|
|
1327
|
+
this.activate(next);
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
const scratchMaterialisations = new ScratchMaterialisationGate();
|
|
1331
|
+
const SCRATCH_LEASE_STALE_MS = 30 * 60_000;
|
|
1332
|
+
const scratchLeaseOwnerPath = (leasePath) => join(leasePath, "owner.json");
|
|
1333
|
+
/**
|
|
1334
|
+
* Processes from different vaults can still spend the same filesystem's free
|
|
1335
|
+
* bytes. Key the well-known lease by the scratch directory's device id in a
|
|
1336
|
+
* per-user temp namespace; fall back to this vault's trusted runtime directory
|
|
1337
|
+
* only on platforms where that identity cannot be established.
|
|
1338
|
+
*/
|
|
1339
|
+
export async function ensurePrivateScratchCoordination(baseDir, user, expectedUid = typeof process.getuid === "function" ? process.getuid() : undefined) {
|
|
1340
|
+
const namespace = join(baseDir, `sealkeep-${user}`);
|
|
1341
|
+
const coordination = join(namespace, "scratch-volume-leases");
|
|
1342
|
+
for (const path of [namespace, coordination]) {
|
|
302
1343
|
try {
|
|
303
|
-
|
|
1344
|
+
await mkdir(path, { mode: 0o700 });
|
|
304
1345
|
}
|
|
305
1346
|
catch (error) {
|
|
306
|
-
|
|
1347
|
+
if (error.code !== "EEXIST")
|
|
1348
|
+
throw error;
|
|
1349
|
+
}
|
|
1350
|
+
const entry = await lstat(path);
|
|
1351
|
+
if (entry.isSymbolicLink() || !entry.isDirectory()) {
|
|
1352
|
+
return fail("internal", `Sealkeep refused an unsafe archive scratch coordination path (${path} is not a real directory).`);
|
|
1353
|
+
}
|
|
1354
|
+
if (expectedUid !== undefined && entry.uid !== expectedUid) {
|
|
1355
|
+
return fail("internal", `Sealkeep refused an archive scratch coordination directory not owned by this user (${path}).`);
|
|
1356
|
+
}
|
|
1357
|
+
if ((entry.mode & 0o777) !== 0o700) {
|
|
1358
|
+
return fail("internal", `Sealkeep refused an archive scratch coordination directory without private 0700 permissions (${path}).`);
|
|
307
1359
|
}
|
|
308
1360
|
}
|
|
309
|
-
return
|
|
1361
|
+
return coordination;
|
|
1362
|
+
}
|
|
1363
|
+
async function scratchLeasePath(dataDir, spaceRoot) {
|
|
1364
|
+
try {
|
|
1365
|
+
const device = (await stat(spaceRoot)).dev;
|
|
1366
|
+
const user = typeof process.getuid === "function" ? String(process.getuid()) : "user";
|
|
1367
|
+
// Never let a predictable /tmp pathname be redirected through a symlink
|
|
1368
|
+
// or pre-created by another local account. Each component below the OS
|
|
1369
|
+
// temp root is a real, current-user-owned 0700 directory before a ticket
|
|
1370
|
+
// or lease token is written into it.
|
|
1371
|
+
const coordination = await ensurePrivateScratchCoordination(tmpdir(), user);
|
|
1372
|
+
return join(coordination, `device-${device}.lock`);
|
|
1373
|
+
}
|
|
1374
|
+
catch (error) {
|
|
1375
|
+
// A per-vault fallback is not safe: two vaults on the same nearly-full
|
|
1376
|
+
// volume would each believe they exclusively reserved the remaining disk.
|
|
1377
|
+
// Fail closed when the shared volume identity/coordination directory is
|
|
1378
|
+
// unavailable; the original provider bytes remain untouched.
|
|
1379
|
+
return fail("internal", `Sealkeep could not reserve this volume's shared archive workspace: ${error instanceof Error ? error.message : "coordination unavailable"}`, { dataDir });
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
function parseScratchLeaseOwner(raw) {
|
|
1383
|
+
if (!raw)
|
|
1384
|
+
return null;
|
|
1385
|
+
try {
|
|
1386
|
+
const value = JSON.parse(raw);
|
|
1387
|
+
if (value.version !== 1 || !Number.isSafeInteger(value.pid) || Number(value.pid) < 1
|
|
1388
|
+
|| typeof value.processIdentity !== "string" || value.processIdentity.length < 1 || value.processIdentity.length > 512
|
|
1389
|
+
|| typeof value.nonce !== "string" || !/^[0-9a-f-]{36}$/i.test(value.nonce)
|
|
1390
|
+
|| typeof value.acquiredAt !== "string")
|
|
1391
|
+
return null;
|
|
1392
|
+
return value;
|
|
1393
|
+
}
|
|
1394
|
+
catch {
|
|
1395
|
+
return null;
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
/**
|
|
1399
|
+
* Separate local servers can perform share/restore/transfer work at once, and
|
|
1400
|
+
* an older index child may still own the v1 lease during an upgrade. The
|
|
1401
|
+
* in-memory gate above cannot make their `statfs` checks atomic. This volume
|
|
1402
|
+
* lease spans download plus consumer use across processes; a dead process
|
|
1403
|
+
* generation is reclaimable, while an unidentifiable live owner fails closed
|
|
1404
|
+
* and keeps its possible scratch bytes reserved.
|
|
1405
|
+
*/
|
|
1406
|
+
async function acquireCrossProcessScratchLease(dataDir, spaceRoot, signal, deadline, waitMs) {
|
|
1407
|
+
if (signal?.aborted)
|
|
1408
|
+
throw cancellationError(signal);
|
|
1409
|
+
const identityBeforeDeadline = async (pid) => {
|
|
1410
|
+
const remaining = deadline - Date.now();
|
|
1411
|
+
if (remaining <= 0)
|
|
1412
|
+
throw scratchBusyError(waitMs);
|
|
1413
|
+
let timer;
|
|
1414
|
+
try {
|
|
1415
|
+
return await Promise.race([
|
|
1416
|
+
daemonProcessIdentity(pid),
|
|
1417
|
+
new Promise((resolve) => {
|
|
1418
|
+
timer = setTimeout(() => resolve({ state: "unknown" }), remaining);
|
|
1419
|
+
}),
|
|
1420
|
+
]);
|
|
1421
|
+
}
|
|
1422
|
+
finally {
|
|
1423
|
+
if (timer)
|
|
1424
|
+
clearTimeout(timer);
|
|
1425
|
+
}
|
|
1426
|
+
};
|
|
1427
|
+
const self = await identityBeforeDeadline(process.pid);
|
|
1428
|
+
if (self.state !== "identified") {
|
|
1429
|
+
return fail("internal", "Sealkeep could not verify this process generation before reserving archive scratch space");
|
|
1430
|
+
}
|
|
1431
|
+
const leasePath = await scratchLeasePath(dataDir, spaceRoot);
|
|
1432
|
+
const runtimeDir = dirname(leasePath);
|
|
1433
|
+
await mkdir(runtimeDir, { recursive: true, mode: 0o700 });
|
|
1434
|
+
const owner = {
|
|
1435
|
+
version: 1,
|
|
1436
|
+
pid: process.pid,
|
|
1437
|
+
processIdentity: self.identity,
|
|
1438
|
+
nonce: randomUUID(),
|
|
1439
|
+
acquiredAt: new Date().toISOString(),
|
|
1440
|
+
};
|
|
1441
|
+
const token = `${JSON.stringify(owner)}\n`;
|
|
1442
|
+
let acquired;
|
|
1443
|
+
// A tiny FIFO ticket stays outside the long-held lease. Unlike repeated
|
|
1444
|
+
// rename races, lexicographic admission cannot indefinitely starve one
|
|
1445
|
+
// process behind newer contenders. The content carries the same process-
|
|
1446
|
+
// generation proof as the lease, so a crashed waiter can be removed safely.
|
|
1447
|
+
const ticketPrefix = `${basename(leasePath)}.ticket-`;
|
|
1448
|
+
const ticketName = `${ticketPrefix}${String(Date.now()).padStart(16, "0")}-${String(process.pid).padStart(10, "0")}-${owner.nonce}.json`;
|
|
1449
|
+
const ticketPath = join(runtimeDir, ticketName);
|
|
1450
|
+
const ticketPending = `${ticketPath}.pending`;
|
|
1451
|
+
try {
|
|
1452
|
+
await writeFile(ticketPending, token, { flag: "wx", mode: 0o600 });
|
|
1453
|
+
await rename(ticketPending, ticketPath);
|
|
1454
|
+
}
|
|
1455
|
+
catch (error) {
|
|
1456
|
+
await unlink(ticketPending).catch(() => undefined);
|
|
1457
|
+
throw error;
|
|
1458
|
+
}
|
|
1459
|
+
const liveTickets = async () => {
|
|
1460
|
+
// Tickets after ours cannot delay us. Avoid process-generation probes for
|
|
1461
|
+
// those newer waiters, and stop the stale-ticket scan at the same deadline
|
|
1462
|
+
// as acquisition so directory litter cannot stretch a five-second bound.
|
|
1463
|
+
const names = (await readdir(runtimeDir))
|
|
1464
|
+
.filter((name) => name.startsWith(ticketPrefix) && name.endsWith(".json") && name <= ticketName)
|
|
1465
|
+
.sort();
|
|
1466
|
+
const live = [];
|
|
1467
|
+
for (const name of names) {
|
|
1468
|
+
if (Date.now() >= deadline)
|
|
1469
|
+
throw scratchBusyError(waitMs);
|
|
1470
|
+
const path = join(runtimeDir, name);
|
|
1471
|
+
const [raw, entry] = await Promise.all([
|
|
1472
|
+
readFile(path, "utf8").catch(() => null),
|
|
1473
|
+
stat(path).catch(() => null),
|
|
1474
|
+
]);
|
|
1475
|
+
if (!entry)
|
|
1476
|
+
continue;
|
|
1477
|
+
if (name === ticketName && raw === token) {
|
|
1478
|
+
live.push(name);
|
|
1479
|
+
continue;
|
|
1480
|
+
}
|
|
1481
|
+
const ticketOwner = parseScratchLeaseOwner(raw);
|
|
1482
|
+
let abandoned = false;
|
|
1483
|
+
if (ticketOwner) {
|
|
1484
|
+
const generation = await identityBeforeDeadline(ticketOwner.pid);
|
|
1485
|
+
abandoned = classifyDaemonProcessGeneration(generation, ticketOwner.processIdentity) === "stale";
|
|
1486
|
+
}
|
|
1487
|
+
else if (Date.now() - entry.mtimeMs > SCRATCH_LEASE_STALE_MS) {
|
|
1488
|
+
abandoned = true;
|
|
1489
|
+
}
|
|
1490
|
+
if (abandoned) {
|
|
1491
|
+
// The user-private coordination directory prevents hostile replacement,
|
|
1492
|
+
// but still compare the complete token so two legitimate rescuers never
|
|
1493
|
+
// unlink a newer ticket that reused the same pathname.
|
|
1494
|
+
if (await readFile(path, "utf8").then((current) => current === raw).catch(() => false)) {
|
|
1495
|
+
await unlink(path).catch(() => undefined);
|
|
1496
|
+
}
|
|
1497
|
+
continue;
|
|
1498
|
+
}
|
|
1499
|
+
live.push(name);
|
|
1500
|
+
}
|
|
1501
|
+
return live;
|
|
1502
|
+
};
|
|
1503
|
+
try {
|
|
1504
|
+
for (;;) {
|
|
1505
|
+
if (signal?.aborted)
|
|
1506
|
+
throw cancellationError(signal);
|
|
1507
|
+
if (Date.now() >= deadline)
|
|
1508
|
+
throw scratchBusyError(waitMs);
|
|
1509
|
+
const tickets = await liveTickets();
|
|
1510
|
+
if (tickets[0] !== ticketName) {
|
|
1511
|
+
await waitForScratchLease(signal, Math.min(50, Math.max(1, deadline - Date.now())));
|
|
1512
|
+
continue;
|
|
1513
|
+
}
|
|
1514
|
+
const staging = `${leasePath}.${owner.nonce}.pending`;
|
|
1515
|
+
try {
|
|
1516
|
+
await mkdir(staging, { mode: 0o700 });
|
|
1517
|
+
await writeFile(scratchLeaseOwnerPath(staging), token, { flag: "wx", mode: 0o600 });
|
|
1518
|
+
const candidate = await stat(staging);
|
|
1519
|
+
await rename(staging, leasePath);
|
|
1520
|
+
acquired = { dev: candidate.dev, ino: candidate.ino, birthtimeMs: candidate.birthtimeMs };
|
|
1521
|
+
break;
|
|
1522
|
+
}
|
|
1523
|
+
catch (error) {
|
|
1524
|
+
await unlink(scratchLeaseOwnerPath(staging)).catch(() => undefined);
|
|
1525
|
+
await rmdir(staging).catch(() => undefined);
|
|
1526
|
+
const code = error.code;
|
|
1527
|
+
const leaseExists = await stat(leasePath).then((entry) => entry.isDirectory()).catch(() => false);
|
|
1528
|
+
// A legitimate owner can release between our failed publish and this
|
|
1529
|
+
// observation. Its now-absent directory is a retry opportunity, not an
|
|
1530
|
+
// archive-transfer error. Never infer the same from a permission error.
|
|
1531
|
+
if (!leaseExists && (code === "EEXIST" || code === "ENOTEMPTY"))
|
|
1532
|
+
continue;
|
|
1533
|
+
if (!leaseExists || (code !== "EEXIST" && code !== "ENOTEMPTY" && code !== "EPERM"))
|
|
1534
|
+
throw error;
|
|
1535
|
+
const [held, raw] = await Promise.all([
|
|
1536
|
+
stat(leasePath).catch(() => null),
|
|
1537
|
+
readFile(scratchLeaseOwnerPath(leasePath), "utf8").catch(() => null),
|
|
1538
|
+
]);
|
|
1539
|
+
const currentOwner = parseScratchLeaseOwner(raw);
|
|
1540
|
+
let abandoned = false;
|
|
1541
|
+
if (currentOwner) {
|
|
1542
|
+
const generation = await identityBeforeDeadline(currentOwner.pid);
|
|
1543
|
+
abandoned = classifyDaemonProcessGeneration(generation, currentOwner.processIdentity) === "stale";
|
|
1544
|
+
}
|
|
1545
|
+
else if (held && Date.now() - held.mtimeMs > SCRATCH_LEASE_STALE_MS) {
|
|
1546
|
+
// Complete lock directories are published by rename, so malformed
|
|
1547
|
+
// content is never a normal acquisition window. Keep it for a generous
|
|
1548
|
+
// transition period before treating old/corrupt state as abandoned.
|
|
1549
|
+
abandoned = true;
|
|
1550
|
+
}
|
|
1551
|
+
if (abandoned && held) {
|
|
1552
|
+
const fingerprint = createHash("sha256")
|
|
1553
|
+
.update(`${held.dev}:${held.ino}:${held.birthtimeMs}:${raw ?? ""}`)
|
|
1554
|
+
.digest("hex")
|
|
1555
|
+
.slice(0, 20);
|
|
1556
|
+
const quarantine = `${leasePath}.stale-${fingerprint}`;
|
|
1557
|
+
const [again, currentRaw] = await Promise.all([
|
|
1558
|
+
stat(leasePath).catch(() => null),
|
|
1559
|
+
readFile(scratchLeaseOwnerPath(leasePath), "utf8").catch(() => null),
|
|
1560
|
+
]);
|
|
1561
|
+
const same = !!again && again.dev === held.dev && again.ino === held.ino
|
|
1562
|
+
&& again.birthtimeMs === held.birthtimeMs && currentRaw === raw;
|
|
1563
|
+
if (same) {
|
|
1564
|
+
try {
|
|
1565
|
+
await rename(leasePath, quarantine);
|
|
1566
|
+
}
|
|
1567
|
+
catch (moveError) {
|
|
1568
|
+
const moveCode = moveError.code;
|
|
1569
|
+
if (moveCode !== "ENOENT" && moveCode !== "EEXIST" && moveCode !== "ENOTEMPTY" && moveCode !== "EPERM")
|
|
1570
|
+
throw moveError;
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
// Deliberately retain the non-empty quarantine. If two rescuers saw
|
|
1574
|
+
// this dead directory, the first can move it and a new owner can then
|
|
1575
|
+
// acquire the public name before the second reaches rename(). The
|
|
1576
|
+
// deterministic occupied destination makes that delayed rename fail
|
|
1577
|
+
// instead of moving the successor's live lease out of the way.
|
|
1578
|
+
continue;
|
|
1579
|
+
}
|
|
1580
|
+
await waitForScratchLease(signal, Math.min(50, Math.max(1, deadline - Date.now())));
|
|
1581
|
+
}
|
|
1582
|
+
}
|
|
1583
|
+
}
|
|
1584
|
+
finally {
|
|
1585
|
+
await unlink(ticketPath).catch(() => undefined);
|
|
1586
|
+
await unlink(ticketPending).catch(() => undefined);
|
|
1587
|
+
}
|
|
1588
|
+
if (!acquired)
|
|
1589
|
+
return fail("internal", "Sealkeep lost its archive scratch reservation while acquiring it");
|
|
1590
|
+
return async () => {
|
|
1591
|
+
const absentOnly = async (operation) => {
|
|
1592
|
+
try {
|
|
1593
|
+
return await operation();
|
|
1594
|
+
}
|
|
1595
|
+
catch (error) {
|
|
1596
|
+
if (error.code === "ENOENT")
|
|
1597
|
+
return null;
|
|
1598
|
+
throw error;
|
|
1599
|
+
}
|
|
1600
|
+
};
|
|
1601
|
+
const [held, raw] = await Promise.all([
|
|
1602
|
+
absentOnly(() => stat(leasePath)),
|
|
1603
|
+
absentOnly(() => readFile(scratchLeaseOwnerPath(leasePath), "utf8")),
|
|
1604
|
+
]);
|
|
1605
|
+
if (!held || held.dev !== acquired.dev || held.ino !== acquired.ino
|
|
1606
|
+
|| held.birthtimeMs !== acquired.birthtimeMs || raw !== token)
|
|
1607
|
+
return;
|
|
1608
|
+
const released = `${leasePath}.released-${owner.nonce}`;
|
|
1609
|
+
try {
|
|
1610
|
+
await rename(leasePath, released);
|
|
1611
|
+
}
|
|
1612
|
+
catch (error) {
|
|
1613
|
+
if (error.code === "ENOENT")
|
|
1614
|
+
return;
|
|
1615
|
+
throw error;
|
|
1616
|
+
}
|
|
1617
|
+
await unlink(scratchLeaseOwnerPath(released)).catch(() => undefined);
|
|
1618
|
+
await rmdir(released).catch(() => undefined);
|
|
1619
|
+
};
|
|
1620
|
+
}
|
|
1621
|
+
async function acquireScratchMaterialisationPermit(dataDir, spaceRoot, signal, scratchWaitMs = DEFAULT_SCRATCH_WAIT_MS) {
|
|
1622
|
+
const waitMs = Math.max(1, Math.min(5 * 60_000, Math.ceil(scratchWaitMs)));
|
|
1623
|
+
const deadline = Date.now() + waitMs;
|
|
1624
|
+
const releaseLocal = await scratchMaterialisations.acquire(signal, deadline, waitMs);
|
|
1625
|
+
try {
|
|
1626
|
+
const releaseCrossProcess = await acquireCrossProcessScratchLease(dataDir, spaceRoot, signal, deadline, waitMs);
|
|
1627
|
+
let localReleased = false;
|
|
1628
|
+
let releaseComplete = false;
|
|
1629
|
+
let releaseInFlight;
|
|
1630
|
+
return () => {
|
|
1631
|
+
if (releaseComplete)
|
|
1632
|
+
return Promise.resolve();
|
|
1633
|
+
if (releaseInFlight)
|
|
1634
|
+
return releaseInFlight;
|
|
1635
|
+
const attempt = (async () => {
|
|
1636
|
+
try {
|
|
1637
|
+
await releaseCrossProcess();
|
|
1638
|
+
releaseComplete = true;
|
|
1639
|
+
}
|
|
1640
|
+
finally {
|
|
1641
|
+
// A failed cross-process rename must not monopolise this process's
|
|
1642
|
+
// FIFO lane. Its exact-owner callback remains retryable, while this
|
|
1643
|
+
// local release happens once and only once.
|
|
1644
|
+
if (!localReleased) {
|
|
1645
|
+
localReleased = true;
|
|
1646
|
+
releaseLocal();
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
})();
|
|
1650
|
+
releaseInFlight = attempt;
|
|
1651
|
+
attempt.then(() => { if (releaseInFlight === attempt)
|
|
1652
|
+
releaseInFlight = undefined; }, () => { if (releaseInFlight === attempt)
|
|
1653
|
+
releaseInFlight = undefined; });
|
|
1654
|
+
return attempt;
|
|
1655
|
+
};
|
|
1656
|
+
}
|
|
1657
|
+
catch (error) {
|
|
1658
|
+
releaseLocal();
|
|
1659
|
+
throw error;
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
/**
|
|
1663
|
+
* Restores the sealed archive blob itself to this machine without decrypting
|
|
1664
|
+
* the session. This is the safe first half of a remote-to-remote move: once a
|
|
1665
|
+
* verified local blob exists, the ordinary target upload can copy it anywhere
|
|
1666
|
+
* without buffering two providers through the browser or risking the source.
|
|
1667
|
+
*/
|
|
1668
|
+
export async function restoreArchiveLocal(dataDir, archiveId, options = {}) {
|
|
1669
|
+
const config = await readConfig(dataDir);
|
|
1670
|
+
const record = (await listArchives(dataDir)).find((item) => item.id === archiveId);
|
|
1671
|
+
if (!record)
|
|
1672
|
+
fail("archive_not_found", `Archive not found: ${archiveId}`, { archiveId });
|
|
1673
|
+
if (!isV2(record))
|
|
1674
|
+
fail("invalid_argument", `Archive ${archiveId} uses format v1. Run: sealkeep migrate`, { archiveId });
|
|
1675
|
+
const present = await stat(record.objectPath).then((entry) => entry.isFile()).catch(() => false);
|
|
1676
|
+
if (present) {
|
|
1677
|
+
const { hashFileRange } = await import("../packages/sealkeep-crypto/src/index.js");
|
|
1678
|
+
const measured = await hashFileRange(record.objectPath, 0, undefined, options.onProgress, options.signal);
|
|
1679
|
+
if (measured.sha256 !== record.cipher.ciphertextSha256) {
|
|
1680
|
+
fail("ciphertext_integrity_failed", `A file already exists at ${record.objectPath}, but it is not archive ${archiveId}. It was left untouched.`, { archiveId, path: record.objectPath });
|
|
1681
|
+
}
|
|
1682
|
+
if (record.offloaded) {
|
|
1683
|
+
await mutateArchiveRecord(config.storage.root, archiveId, (current) => {
|
|
1684
|
+
if (!isV2(current))
|
|
1685
|
+
return current;
|
|
1686
|
+
const { offloaded: _offloaded, ...withLocalCopy } = current;
|
|
1687
|
+
return withLocalCopy;
|
|
1688
|
+
});
|
|
1689
|
+
}
|
|
1690
|
+
const bytes = measured.bytes;
|
|
1691
|
+
return { restored: false, alreadyPresent: true, bytes, path: record.objectPath };
|
|
1692
|
+
}
|
|
1693
|
+
const copies = archiveCopies(record);
|
|
1694
|
+
const chosen = options.copyId
|
|
1695
|
+
? copies.find((copy) => copy.copyId === options.copyId)
|
|
1696
|
+
: copies.find((copy) => copy.verifiedAt);
|
|
1697
|
+
if (!chosen) {
|
|
1698
|
+
fail("archive_not_found", options.copyId
|
|
1699
|
+
? `Archive ${archiveId} has no stored copy ${options.copyId}`
|
|
1700
|
+
: `Archive ${archiveId} has no verified stored copy to restore`, { archiveId, copyId: options.copyId });
|
|
1701
|
+
}
|
|
1702
|
+
const { freeBytes, resolveReserveBytes } = await import("./disk.js");
|
|
1703
|
+
const [free, reserve] = await Promise.all([
|
|
1704
|
+
freeBytes(dirname(record.objectPath)),
|
|
1705
|
+
resolveReserveBytes(dataDir),
|
|
1706
|
+
]);
|
|
1707
|
+
const needed = record.cipher.storedBytes;
|
|
1708
|
+
if (free !== undefined && free < needed + reserve) {
|
|
1709
|
+
const gb = (value) => `${(value / 1024 ** 3).toFixed(1)} GB`;
|
|
1710
|
+
fail("insufficient_disk_space", `Restoring archive ${archiveId.slice(0, 8)} needs about ${gb(needed)} plus ${gb(reserve)} of working room; this disk has ${gb(free)} free. Nothing was downloaded.`, { archiveId, free, needed, reserve });
|
|
1711
|
+
}
|
|
1712
|
+
await mkdir(dirname(record.objectPath), { recursive: true, mode: 0o700 });
|
|
1713
|
+
const temporary = `${record.objectPath}.${randomUUID()}.restore`;
|
|
1714
|
+
let writtenBytes = 0;
|
|
1715
|
+
try {
|
|
1716
|
+
// Provider bytes land directly in a private temporary file. Hashing is in
|
|
1717
|
+
// the same write loop, so no complete archive exists in memory and an
|
|
1718
|
+
// invalid response never reaches the canonical pathname.
|
|
1719
|
+
const measured = await writeChunksToFile(temporary, await fetchCiphertextStream(dataDir, record, options.client, chosen, options.overrides, options.signal), { mode: 0o600, expectedBytes: record.cipher.storedBytes, signal: options.signal, onProgress: options.onProgress });
|
|
1720
|
+
if (measured.sha256Hex !== record.cipher.ciphertextSha256 || measured.bytes !== record.cipher.storedBytes) {
|
|
1721
|
+
fail("ciphertext_integrity_failed", `Stored copy ${chosen.copyId} is not archive ${archiveId}; nothing was written`, { archiveId, copyId: chosen.copyId });
|
|
1722
|
+
}
|
|
1723
|
+
writtenBytes = measured.bytes;
|
|
1724
|
+
await rename(temporary, record.objectPath);
|
|
1725
|
+
await mutateArchiveRecord(config.storage.root, archiveId, (current) => {
|
|
1726
|
+
if (!isV2(current))
|
|
1727
|
+
return current;
|
|
1728
|
+
const { offloaded: _offloaded, ...withLocalCopy } = current;
|
|
1729
|
+
return withLocalCopy;
|
|
1730
|
+
});
|
|
1731
|
+
}
|
|
1732
|
+
catch (error) {
|
|
1733
|
+
await rm(temporary, { force: true }).catch(() => undefined);
|
|
1734
|
+
throw error;
|
|
1735
|
+
}
|
|
1736
|
+
await recordAudit(dataDir, "archive.restore_local", "allowed", { archiveId, copyId: chosen.copyId ?? null, bytes: writtenBytes }).catch(() => undefined);
|
|
1737
|
+
return { restored: true, alreadyPresent: false, bytes: writtenBytes, path: record.objectPath };
|
|
1738
|
+
}
|
|
1739
|
+
/**
|
|
1740
|
+
* Restore one logical session's complete encrypted delta chain.
|
|
1741
|
+
*
|
|
1742
|
+
* Every missing link is downloaded into a private sibling temporary file and
|
|
1743
|
+
* checked against its authenticated ciphertext length and digest first. Only
|
|
1744
|
+
* after the *whole* chain passes are the files hard-linked into their canonical
|
|
1745
|
+
* paths. A hard link gives us an exclusive publish operation: another process
|
|
1746
|
+
* cannot race this restore into overwriting an existing archive. If any publish
|
|
1747
|
+
* fails, every path published by this call is removed again, while all remote
|
|
1748
|
+
* copies and archive records remain untouched.
|
|
1749
|
+
*
|
|
1750
|
+
* Metadata is changed only after every required local file exists. A metadata
|
|
1751
|
+
* write failure deliberately keeps the complete verified local chain as an
|
|
1752
|
+
* extra safe copy and leaves all remote copies in place; it can never expose a
|
|
1753
|
+
* head without its base or authorize a provider delete.
|
|
1754
|
+
*/
|
|
1755
|
+
export async function restoreArchiveChainLocal(dataDir, archiveId, options = {}) {
|
|
1756
|
+
const config = await readConfig(dataDir);
|
|
1757
|
+
const archives = await listArchives(dataDir);
|
|
1758
|
+
const record = archives.find((item) => item.id === archiveId);
|
|
1759
|
+
if (!record)
|
|
1760
|
+
fail("archive_not_found", `Archive not found: ${archiveId}`, { archiveId });
|
|
1761
|
+
if (!isV2(record))
|
|
1762
|
+
fail("invalid_argument", `Archive ${archiveId} uses format v1. Run: sealkeep migrate`, { archiveId });
|
|
1763
|
+
const chain = resolveDeltaChain(archives, record);
|
|
1764
|
+
const alreadyPresent = [];
|
|
1765
|
+
const missing = [];
|
|
1766
|
+
// Refuse conflicting local paths before making a provider request. A
|
|
1767
|
+
// same-name file is not evidence that a chain dependency is present.
|
|
1768
|
+
for (const item of chain) {
|
|
1769
|
+
if (!isV2(item))
|
|
1770
|
+
fail("invalid_argument", `Archive ${item.id} uses format v1. Run: sealkeep migrate`, { archiveId: item.id });
|
|
1771
|
+
const present = await stat(item.objectPath).then((entry) => entry.isFile()).catch(() => false);
|
|
1772
|
+
if (present) {
|
|
1773
|
+
const { hashFileRange } = await import("../packages/sealkeep-crypto/src/index.js");
|
|
1774
|
+
const measured = await hashFileRange(item.objectPath);
|
|
1775
|
+
if (measured.bytes !== item.cipher.storedBytes || measured.sha256 !== item.cipher.ciphertextSha256) {
|
|
1776
|
+
fail("ciphertext_integrity_failed", `A file already exists at ${item.objectPath}, but it is not archive ${item.id}. It was left untouched.`, {
|
|
1777
|
+
archiveId: item.id,
|
|
1778
|
+
path: item.objectPath,
|
|
1779
|
+
deltaDependency: item.id !== archiveId,
|
|
1780
|
+
});
|
|
1781
|
+
}
|
|
1782
|
+
alreadyPresent.push(item);
|
|
1783
|
+
continue;
|
|
1784
|
+
}
|
|
1785
|
+
const requestedCopyId = options.copyIds?.[item.id]
|
|
1786
|
+
?? (item.id === archiveId ? options.copyId : undefined);
|
|
1787
|
+
const copies = archiveCopies(item);
|
|
1788
|
+
const chosen = requestedCopyId
|
|
1789
|
+
? copies.find((copy) => copy.copyId === requestedCopyId && copy.verifiedAt)
|
|
1790
|
+
: copies.find((copy) => copy.verifiedAt);
|
|
1791
|
+
if (!chosen) {
|
|
1792
|
+
fail("archive_not_found", requestedCopyId
|
|
1793
|
+
? `Archive ${item.id} no longer has verified stored copy ${requestedCopyId}; no local chain was published.`
|
|
1794
|
+
: `Archive ${item.id} has no verified stored copy; no local chain was published.`, {
|
|
1795
|
+
archiveId: item.id,
|
|
1796
|
+
copyId: requestedCopyId,
|
|
1797
|
+
deltaDependency: item.id !== archiveId,
|
|
1798
|
+
});
|
|
1799
|
+
}
|
|
1800
|
+
missing.push({
|
|
1801
|
+
record: item,
|
|
1802
|
+
copy: chosen,
|
|
1803
|
+
temporary: `${item.objectPath}.${randomUUID()}.restore`,
|
|
1804
|
+
});
|
|
1805
|
+
}
|
|
1806
|
+
const needed = missing.reduce((sum, item) => sum + item.record.cipher.storedBytes, 0);
|
|
1807
|
+
if (needed > 0) {
|
|
1808
|
+
const { freeBytes, resolveReserveBytes } = await import("./disk.js");
|
|
1809
|
+
const [free, reserve] = await Promise.all([
|
|
1810
|
+
freeBytes(dirname(record.objectPath)),
|
|
1811
|
+
resolveReserveBytes(dataDir),
|
|
1812
|
+
]);
|
|
1813
|
+
if (free !== undefined && free < needed + reserve) {
|
|
1814
|
+
const gb = (value) => `${(value / 1024 ** 3).toFixed(1)} GB`;
|
|
1815
|
+
fail("insufficient_disk_space", `Restoring this ${chain.length}-link session chain needs about ${gb(needed)} plus ${gb(reserve)} of working room; this disk has ${gb(free)} free. Nothing was downloaded.`, { archiveId, chain: chain.map((item) => item.id), free, needed, reserve });
|
|
1816
|
+
}
|
|
1817
|
+
}
|
|
1818
|
+
const staged = [];
|
|
1819
|
+
const published = [];
|
|
1820
|
+
try {
|
|
1821
|
+
// Stage and prove every byte before publishing even the oldest base.
|
|
1822
|
+
for (const item of missing) {
|
|
1823
|
+
await mkdir(dirname(item.record.objectPath), { recursive: true, mode: 0o700 });
|
|
1824
|
+
const measured = await writeChunksToFile(item.temporary, await fetchCiphertextStream(dataDir, item.record, options.client, item.copy, options.overrides), { mode: 0o600, expectedBytes: item.record.cipher.storedBytes });
|
|
1825
|
+
if (measured.sha256Hex !== item.record.cipher.ciphertextSha256
|
|
1826
|
+
|| measured.bytes !== item.record.cipher.storedBytes) {
|
|
1827
|
+
fail("ciphertext_integrity_failed", `Stored copy ${item.copy.copyId} is not archive ${item.record.id}; no local chain was published.`, {
|
|
1828
|
+
archiveId: item.record.id,
|
|
1829
|
+
copyId: item.copy.copyId,
|
|
1830
|
+
deltaDependency: item.record.id !== archiveId,
|
|
1831
|
+
});
|
|
1832
|
+
}
|
|
1833
|
+
staged.push(item);
|
|
1834
|
+
}
|
|
1835
|
+
// Hard-link publication is exclusive and cannot overwrite a path that
|
|
1836
|
+
// appeared after the preflight. All links live beside their destinations,
|
|
1837
|
+
// so this never crosses a filesystem boundary.
|
|
1838
|
+
for (const item of staged) {
|
|
1839
|
+
await hardLink(item.temporary, item.record.objectPath);
|
|
1840
|
+
published.push(item);
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
catch (error) {
|
|
1844
|
+
for (const item of published.reverse())
|
|
1845
|
+
await rm(item.record.objectPath, { force: true }).catch(() => undefined);
|
|
1846
|
+
throw error;
|
|
1847
|
+
}
|
|
1848
|
+
finally {
|
|
1849
|
+
for (const item of missing)
|
|
1850
|
+
await rm(item.temporary, { force: true }).catch(() => undefined);
|
|
1851
|
+
}
|
|
1852
|
+
// Every link is now physically present and verified. Clear stale
|
|
1853
|
+
// remote-only locators without forgetting any durable remote copy.
|
|
1854
|
+
for (const item of chain) {
|
|
1855
|
+
if (!isV2(item) || !item.offloaded)
|
|
1856
|
+
continue;
|
|
1857
|
+
await mutateArchiveRecord(config.storage.root, item.id, (current) => {
|
|
1858
|
+
if (!isV2(current))
|
|
1859
|
+
return current;
|
|
1860
|
+
const { offloaded: _offloaded, ...withLocalCopy } = current;
|
|
1861
|
+
return withLocalCopy;
|
|
1862
|
+
});
|
|
1863
|
+
}
|
|
1864
|
+
const restoredArchiveIds = missing.map((item) => item.record.id);
|
|
1865
|
+
const alreadyPresentArchiveIds = alreadyPresent.map((item) => item.id);
|
|
1866
|
+
const totalBytes = chain.reduce((sum, item) => sum + (isV2(item) ? item.cipher.storedBytes : 0), 0);
|
|
1867
|
+
const restoredBytes = missing.reduce((sum, item) => sum + item.record.cipher.storedBytes, 0);
|
|
1868
|
+
await recordAudit(dataDir, "archive.restore_local", "allowed", {
|
|
1869
|
+
archiveId,
|
|
1870
|
+
chainLength: chain.length,
|
|
1871
|
+
restoredCount: restoredArchiveIds.length,
|
|
1872
|
+
alreadyPresentCount: alreadyPresentArchiveIds.length,
|
|
1873
|
+
bytes: restoredBytes,
|
|
1874
|
+
}).catch(() => undefined);
|
|
1875
|
+
return {
|
|
1876
|
+
restored: missing.length > 0,
|
|
1877
|
+
alreadyPresent: missing.length === 0,
|
|
1878
|
+
bytes: totalBytes,
|
|
1879
|
+
restoredBytes,
|
|
1880
|
+
path: record.objectPath,
|
|
1881
|
+
chain: chain.map((item) => item.id),
|
|
1882
|
+
restoredArchiveIds,
|
|
1883
|
+
alreadyPresentArchiveIds,
|
|
1884
|
+
};
|
|
310
1885
|
}
|
|
311
1886
|
/**
|
|
312
1887
|
* Gives back a local path holding this archive's ciphertext, fetching it first
|
|
@@ -317,28 +1892,161 @@ export async function checkRemoteCopyMany(dataDir, ids, client) {
|
|
|
317
1892
|
* one and removed afterwards, which keeps the disk saving intact: the bytes are
|
|
318
1893
|
* on this machine only for as long as the restore takes.
|
|
319
1894
|
*/
|
|
320
|
-
export async function materialiseArchive(dataDir, record, client) {
|
|
321
|
-
if (!isV2(record) || !record.offloaded)
|
|
1895
|
+
export async function materialiseArchive(dataDir, record, client, options = {}) {
|
|
1896
|
+
if (!isV2(record) || (!record.offloaded && !options.copyId))
|
|
322
1897
|
return { path: record.objectPath, release: async () => { } };
|
|
1898
|
+
const selected = options.copyId
|
|
1899
|
+
? archiveCopies(record).find((copy) => copy.copyId === options.copyId)
|
|
1900
|
+
: archiveCopies(record).find((copy) => copy.provider === record.offloaded?.provider
|
|
1901
|
+
&& copy.bucket === record.offloaded.bucket
|
|
1902
|
+
&& copy.objectKey === record.offloaded.objectKey);
|
|
1903
|
+
if (!selected) {
|
|
1904
|
+
fail("archive_not_found", options.copyId
|
|
1905
|
+
? `Archive ${record.id} has no stored copy ${options.copyId}`
|
|
1906
|
+
: `Archive ${record.id} has no stored copy matching its offloaded locator`, { archiveId: record.id, copyId: options.copyId });
|
|
1907
|
+
}
|
|
1908
|
+
const ownedPath = options.path;
|
|
1909
|
+
if (ownedPath) {
|
|
1910
|
+
const existing = await stat(ownedPath).then((entry) => entry.isFile()).catch(() => false);
|
|
1911
|
+
if (existing) {
|
|
1912
|
+
const { hashFileRange } = await import("../packages/sealkeep-crypto/src/index.js");
|
|
1913
|
+
const measured = await hashFileRange(ownedPath, 0, undefined, options.onProgress, options.signal);
|
|
1914
|
+
if (measured.sha256 !== record.cipher.ciphertextSha256) {
|
|
1915
|
+
fail("ciphertext_integrity_failed", `The transfer staging file for archive ${record.id} is not the recorded ciphertext. It was left untouched.`, { archiveId: record.id, path: ownedPath });
|
|
1916
|
+
}
|
|
1917
|
+
return {
|
|
1918
|
+
path: ownedPath,
|
|
1919
|
+
release: async () => {
|
|
1920
|
+
await rm(ownedPath, { force: true });
|
|
1921
|
+
await rm(`${ownedPath}.partial`, { force: true });
|
|
1922
|
+
},
|
|
1923
|
+
};
|
|
1924
|
+
}
|
|
1925
|
+
}
|
|
1926
|
+
const spaceRoot = ownedPath ? dirname(ownedPath) : options.scratchRoot ?? tmpdir();
|
|
1927
|
+
if (ownedPath)
|
|
1928
|
+
await mkdir(spaceRoot, { recursive: true, mode: 0o700 });
|
|
1929
|
+
// Claim the shared scratch lane BEFORE measuring free space. Holding this
|
|
1930
|
+
// permit through `release()` makes the measurement and the archive-sized
|
|
1931
|
+
// lifetime one atomic decision with respect to parent and child processes.
|
|
1932
|
+
const releaseScratchPermit = await acquireScratchMaterialisationPermit(dataDir, spaceRoot, options.signal, options.scratchWaitMs);
|
|
1933
|
+
let permitReleaseComplete = false;
|
|
1934
|
+
let permitReleaseInFlight;
|
|
1935
|
+
const relinquishPermit = () => {
|
|
1936
|
+
if (permitReleaseComplete)
|
|
1937
|
+
return Promise.resolve();
|
|
1938
|
+
if (permitReleaseInFlight)
|
|
1939
|
+
return permitReleaseInFlight;
|
|
1940
|
+
const attempt = (async () => { await releaseScratchPermit(); })();
|
|
1941
|
+
permitReleaseInFlight = attempt;
|
|
1942
|
+
attempt.then(() => {
|
|
1943
|
+
permitReleaseComplete = true;
|
|
1944
|
+
if (permitReleaseInFlight === attempt)
|
|
1945
|
+
permitReleaseInFlight = undefined;
|
|
1946
|
+
}, () => { if (permitReleaseInFlight === attempt)
|
|
1947
|
+
permitReleaseInFlight = undefined; });
|
|
1948
|
+
return attempt;
|
|
1949
|
+
};
|
|
323
1950
|
// Reading an offloaded archive means writing it back to disk in full, and
|
|
324
1951
|
// this runs in a loop during an index build. On a machine that offloaded
|
|
325
1952
|
// BECAUSE it was short of space, that filled the disk and died on ENOSPC
|
|
326
1953
|
// mid-build — the exact failure this product exists to prevent. Ask first,
|
|
327
1954
|
// and refuse in words that name the archive and the shortfall.
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
1955
|
+
let scratch;
|
|
1956
|
+
let path = ownedPath ?? "";
|
|
1957
|
+
let partial = "";
|
|
1958
|
+
let cleanFiles;
|
|
1959
|
+
try {
|
|
1960
|
+
if (options.signal?.aborted)
|
|
1961
|
+
throw cancellationError(options.signal);
|
|
1962
|
+
const { freeBytes, resolveReserveBytes } = await import("./disk.js");
|
|
1963
|
+
const [free, reserve] = await Promise.all([freeBytes(spaceRoot), resolveReserveBytes(dataDir)]);
|
|
1964
|
+
const needed = record.cipher.storedBytes ?? record.remote?.bytes ?? 0;
|
|
1965
|
+
if (free !== undefined && needed > 0 && free < needed + reserve) {
|
|
1966
|
+
const gb = (n) => `${(n / 1024 ** 3).toFixed(1)} GB`;
|
|
1967
|
+
fail("insufficient_disk_space", `Reading archive ${record.id.slice(0, 8)} needs about ${gb(needed)} of temporary space plus ${gb(reserve)} kept free for this machine; only ${gb(free)} is available. Nothing was written. Free space before trying this operation again.`, { archiveId: record.id, free, needed, reserve });
|
|
1968
|
+
}
|
|
1969
|
+
scratch = ownedPath ? undefined : await mkdtemp(join(spaceRoot, "sealkeep-fetch-"));
|
|
1970
|
+
path = ownedPath ?? join(scratch, `${record.id}.skarchive`);
|
|
1971
|
+
partial = `${path}.partial`;
|
|
1972
|
+
cleanFiles = ownedPath
|
|
1973
|
+
? async () => {
|
|
1974
|
+
await rm(path, { force: true });
|
|
1975
|
+
await rm(partial, { force: true });
|
|
1976
|
+
}
|
|
1977
|
+
: async () => { await rm(scratch, { recursive: true, force: true }); };
|
|
1978
|
+
// A deterministic partial belongs to the same journaled owner as `path`.
|
|
1979
|
+
// A crash may leave it behind; the next attempt removes exactly that file
|
|
1980
|
+
// before writing, and a successful rename never exposes partial bytes.
|
|
1981
|
+
await rm(partial, { force: true });
|
|
1982
|
+
const downloaded = await fetchCiphertextStream(dataDir, record, client, selected, options.overrides, options.signal, undefined, options.maxBytesPerSecond);
|
|
1983
|
+
const cooperative = (async function* () {
|
|
1984
|
+
for await (const chunk of downloaded) {
|
|
1985
|
+
yield chunk;
|
|
1986
|
+
await options.onYield?.();
|
|
1987
|
+
}
|
|
1988
|
+
})();
|
|
1989
|
+
const measured = await writeChunksToFile(partial, cooperative, { mode: 0o600, signal: options.signal, expectedBytes: record.cipher.storedBytes, onProgress: options.onProgress });
|
|
1990
|
+
if (measured.sha256Hex !== record.cipher.ciphertextSha256 || measured.bytes !== record.cipher.storedBytes) {
|
|
1991
|
+
fail("ciphertext_integrity_failed", `The copy of archive ${record.id} in ${selected.bucket} is not the archive that was stored there`, { archiveId: record.id, copyId: selected.copyId });
|
|
1992
|
+
}
|
|
1993
|
+
await rename(partial, path);
|
|
1994
|
+
let releaseComplete = false;
|
|
1995
|
+
let releaseInFlight;
|
|
1996
|
+
return {
|
|
1997
|
+
path,
|
|
1998
|
+
release: () => {
|
|
1999
|
+
if (releaseComplete)
|
|
2000
|
+
return Promise.resolve();
|
|
2001
|
+
if (releaseInFlight)
|
|
2002
|
+
return releaseInFlight;
|
|
2003
|
+
const attempt = (async () => {
|
|
2004
|
+
// Deleting the bytes is the admission boundary. If cleanup fails,
|
|
2005
|
+
// retain both the in-process lane and cross-process lease so another
|
|
2006
|
+
// archive cannot be admitted on top of space that still exists. The
|
|
2007
|
+
// idempotent release handle lets its owner retry the exact cleanup.
|
|
2008
|
+
await cleanFiles();
|
|
2009
|
+
await relinquishPermit();
|
|
2010
|
+
})();
|
|
2011
|
+
releaseInFlight = attempt;
|
|
2012
|
+
attempt.then(() => {
|
|
2013
|
+
releaseComplete = true;
|
|
2014
|
+
if (releaseInFlight === attempt)
|
|
2015
|
+
releaseInFlight = undefined;
|
|
2016
|
+
}, () => { if (releaseInFlight === attempt)
|
|
2017
|
+
releaseInFlight = undefined; });
|
|
2018
|
+
return attempt;
|
|
2019
|
+
},
|
|
2020
|
+
};
|
|
2021
|
+
}
|
|
2022
|
+
catch (error) {
|
|
2023
|
+
let cleanupError;
|
|
2024
|
+
try {
|
|
2025
|
+
if (cleanFiles)
|
|
2026
|
+
await cleanFiles();
|
|
2027
|
+
else {
|
|
2028
|
+
if (partial)
|
|
2029
|
+
await rm(partial, { force: true });
|
|
2030
|
+
if (scratch)
|
|
2031
|
+
await rm(scratch, { recursive: true, force: true });
|
|
2032
|
+
}
|
|
2033
|
+
}
|
|
2034
|
+
catch (cleanupFailure) {
|
|
2035
|
+
cleanupError = cleanupFailure;
|
|
2036
|
+
}
|
|
2037
|
+
if (cleanupError) {
|
|
2038
|
+
// There is no materialised handle to return on a failed fetch, so fail
|
|
2039
|
+
// visibly and keep the reservation for the lifetime of this process.
|
|
2040
|
+
// Releasing it while unknown scratch bytes remain would violate the disk
|
|
2041
|
+
// reserve; a service restart makes the exact-generation lease reclaimable.
|
|
2042
|
+
throw withScratchReleaseFailure(error, cleanupError);
|
|
2043
|
+
}
|
|
2044
|
+
try {
|
|
2045
|
+
await relinquishPermit();
|
|
2046
|
+
}
|
|
2047
|
+
catch (releaseError) {
|
|
2048
|
+
throw withScratchReleaseFailure(error, releaseError);
|
|
2049
|
+
}
|
|
2050
|
+
throw error;
|
|
2051
|
+
}
|
|
344
2052
|
}
|