sealkeep 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +163 -14
- package/CHANGELOG.md +223 -1
- package/CONTROL_PLANE.md +2 -2
- package/LICENSE +1 -1
- package/README.md +108 -22
- package/THIRD_PARTY.md +2 -2
- package/THREAT_MODEL.md +23 -4
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/chunk-access.d.ts +26 -4
- package/dist/packages/sealkeep-crypto/src/chunk-access.js +219 -0
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/envelope.d.ts +11 -1
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/envelope.js +58 -21
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/format.d.ts +2 -2
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/index.d.ts +1 -0
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/index.js +1 -0
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/recipients.js +4 -4
- package/dist/packages/sealkeep-crypto/src/squeeze.d.ts +6 -0
- package/dist/packages/sealkeep-crypto/src/squeeze.js +39 -0
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/stream.d.ts +36 -3
- package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/stream.js +196 -37
- package/dist/site/index.html +1808 -1904
- package/dist/site/llms.txt +67 -0
- package/dist/site/trust/architecture-data-flow.html +53 -0
- package/dist/site/trust/audit-roadmap.html +37 -0
- package/dist/site/trust/deployment-responsibility.html +11 -0
- package/dist/site/trust/dpa-sample.html +30 -0
- package/dist/site/trust/release-provenance.html +21 -0
- package/dist/site/trust/subprocessors.html +15 -0
- package/dist/site/trust/threat-model.html +34 -0
- package/dist/site/trust/toms.html +41 -0
- package/dist/site/trust-document.css +32 -0
- package/dist/site/trust.html +73 -0
- package/dist/site/visual/assets/index-8Nxnkc7o.js +207 -0
- package/dist/site/visual/assets/index-oxLy1bpy.css +1 -0
- package/dist/site/visual/index.html +18 -0
- package/dist/site.zip +0 -0
- package/dist/src/activity.d.ts +9 -0
- package/dist/src/activity.js +90 -1
- package/dist/src/adapters.d.ts +175 -5
- package/dist/src/adapters.js +961 -73
- package/dist/src/agent-context.d.ts +135 -0
- package/dist/src/agent-context.js +1059 -0
- package/dist/src/archive-copies.d.ts +47 -0
- package/dist/src/archive-copies.js +179 -0
- package/dist/src/audit.d.ts +1 -1
- package/dist/src/audit.js +29 -4
- package/dist/src/autopilot.d.ts +52 -7
- package/dist/src/autopilot.js +143 -25
- package/dist/src/background-bandwidth.d.ts +46 -0
- package/dist/src/background-bandwidth.js +301 -0
- package/dist/src/background-cpu.d.ts +82 -0
- package/dist/src/background-cpu.js +212 -0
- package/dist/src/background-worker-error.d.ts +12 -0
- package/dist/src/background-worker-error.js +18 -0
- package/dist/src/branding.d.ts +24 -1
- package/dist/src/branding.js +31 -1
- package/dist/src/bridge.d.ts +233 -0
- package/dist/src/bridge.js +604 -0
- package/dist/src/byte-stream.d.ts +91 -0
- package/dist/src/byte-stream.js +385 -0
- package/dist/src/chunk-store.d.ts +41 -8
- package/dist/src/chunk-store.js +161 -65
- package/dist/src/cli.js +1698 -163
- package/dist/src/cloud.d.ts +841 -31
- package/dist/src/cloud.js +3196 -277
- package/dist/src/context-background.d.ts +37 -0
- package/dist/src/context-background.js +309 -0
- package/dist/src/context-drain-child.d.ts +1 -0
- package/dist/src/context-drain-child.js +98 -0
- package/dist/src/context-reader.d.ts +118 -0
- package/dist/src/context-reader.js +447 -0
- package/dist/src/control-plane/auth.d.ts +32 -4
- package/dist/src/control-plane/auth.js +85 -24
- package/dist/src/control-plane/server.js +19 -6
- package/dist/src/control-plane.d.ts +17 -1
- package/dist/src/control-plane.js +32 -6
- package/dist/src/crypto.d.ts +1 -1
- package/dist/src/crypto.js +5 -5
- package/dist/src/daemon-lease.d.ts +70 -0
- package/dist/src/daemon-lease.js +420 -0
- package/dist/src/daemon.d.ts +94 -1
- package/dist/src/daemon.js +1082 -105
- package/dist/src/darwin-service-policy.d.ts +41 -0
- package/dist/src/darwin-service-policy.js +60 -0
- package/dist/src/dashboard-cli.js +15 -15
- package/dist/src/device-authorization.d.ts +37 -0
- package/dist/src/device-authorization.js +199 -0
- package/dist/src/device-enrollment.d.ts +91 -0
- package/dist/src/device-enrollment.js +349 -0
- package/dist/src/disk.d.ts +17 -12
- package/dist/src/disk.js +43 -17
- package/dist/src/doctor.d.ts +35 -1
- package/dist/src/doctor.js +316 -41
- package/dist/src/durable-ticket-lock.d.ts +24 -0
- package/dist/src/durable-ticket-lock.js +232 -0
- package/dist/src/enroll.d.ts +1 -1
- package/dist/src/enroll.js +13 -7
- package/dist/src/env.d.ts +10 -1
- package/dist/src/env.js +11 -3
- package/dist/src/errors.d.ts +8 -8
- package/dist/src/errors.js +6 -6
- package/dist/src/flush.d.ts +12 -0
- package/dist/src/flush.js +37 -0
- package/dist/src/heartbeat.d.ts +86 -12
- package/dist/src/heartbeat.js +415 -29
- package/dist/src/index-background-watchdog.d.ts +1 -0
- package/dist/src/index-background-watchdog.js +94 -0
- package/dist/src/index-background-work.d.ts +21 -0
- package/dist/src/index-background-work.js +25 -0
- package/dist/src/index-background.d.ts +64 -0
- package/dist/src/index-background.js +394 -0
- package/dist/src/index-build-child.d.ts +1 -0
- package/dist/src/index-build-child.js +109 -0
- package/dist/src/index-manifest.d.ts +52 -0
- package/dist/src/index-manifest.js +444 -0
- package/dist/src/index-publication-proof.d.ts +84 -0
- package/dist/src/index-publication-proof.js +380 -0
- package/dist/src/index-publication-state.d.ts +149 -0
- package/dist/src/index-publication-state.js +696 -0
- package/dist/src/index-publication-verifier.d.ts +89 -0
- package/dist/src/index-publication-verifier.js +341 -0
- package/dist/src/index-publish.d.ts +62 -0
- package/dist/src/index-publish.js +540 -0
- package/dist/src/index-scratch-cleanup.d.ts +19 -0
- package/dist/src/index-scratch-cleanup.js +166 -0
- package/dist/src/index-segment-types.d.ts +132 -0
- package/dist/src/index-segment-types.js +21 -0
- package/dist/src/index-segments.d.ts +9 -0
- package/dist/src/index-segments.js +516 -0
- package/dist/src/index-store.d.ts +123 -0
- package/dist/src/index-store.js +495 -0
- package/dist/src/index-sync.d.ts +91 -15
- package/dist/src/index-sync.js +286 -53
- package/dist/src/index-upgrade-publication.d.ts +30 -0
- package/dist/src/index-upgrade-publication.js +179 -0
- package/dist/src/integration-manager.d.ts +32 -0
- package/dist/src/integration-manager.js +394 -0
- package/dist/src/leakscan.js +1 -1
- package/dist/src/local-api.d.ts +56 -15
- package/dist/src/local-api.js +4979 -541
- package/dist/src/machine-settings.d.ts +51 -0
- package/dist/src/machine-settings.js +166 -0
- package/dist/src/managed-chunks.d.ts +5 -2
- package/dist/src/managed-chunks.js +14 -14
- package/dist/src/mcp-install.d.ts +11 -9
- package/dist/src/mcp-install.js +73 -28
- package/dist/src/mcp-workspace.d.ts +18 -0
- package/dist/src/mcp-workspace.js +50 -0
- package/dist/src/mcp.js +294 -25
- package/dist/src/migrate.js +27 -21
- package/dist/src/notify.d.ts +1 -1
- package/dist/src/notify.js +5 -5
- package/dist/src/offload.d.ts +201 -14
- package/dist/src/offload.js +1848 -140
- package/dist/src/onboarding.d.ts +8 -1
- package/dist/src/onboarding.js +4 -4
- package/dist/src/packages.d.ts +2 -2
- package/dist/src/packages.js +10 -2
- package/dist/src/passkey.d.ts +0 -1
- package/dist/src/passkey.js +2 -7
- package/dist/src/password-lock.d.ts +2 -2
- package/dist/src/password-lock.js +6 -6
- package/dist/src/paths.d.ts +2 -0
- package/dist/src/paths.js +2 -0
- package/dist/src/presence.d.ts +86 -0
- package/dist/src/presence.js +240 -0
- package/dist/src/progress-deadline.d.ts +21 -0
- package/dist/src/progress-deadline.js +91 -0
- package/dist/src/project-repair.d.ts +55 -0
- package/dist/src/project-repair.js +131 -0
- package/dist/src/providers/gcs.d.ts +28 -7
- package/dist/src/providers/gcs.js +35 -24
- package/dist/src/providers/gdrive.d.ts +71 -8
- package/dist/src/providers/gdrive.js +223 -50
- package/dist/src/providers/index.d.ts +11 -3
- package/dist/src/providers/index.js +30 -10
- package/dist/src/providers/s3.d.ts +30 -8
- package/dist/src/providers/s3.js +41 -30
- package/dist/src/providers/safe-storage-fetch.d.ts +12 -0
- package/dist/src/providers/safe-storage-fetch.js +72 -0
- package/dist/src/queue.d.ts +136 -19
- package/dist/src/queue.js +862 -96
- package/dist/src/reclaim-transaction.d.ts +156 -0
- package/dist/src/reclaim-transaction.js +1027 -0
- package/dist/src/recovery-codes.d.ts +32 -0
- package/dist/src/recovery-codes.js +338 -0
- package/dist/src/recovery.js +12 -9
- package/dist/src/rehydrate.d.ts +25 -22
- package/dist/src/rehydrate.js +319 -23
- package/dist/src/restore.d.ts +57 -4
- package/dist/src/restore.js +272 -36
- package/dist/src/resume-stub.d.ts +92 -0
- package/dist/src/resume-stub.js +417 -0
- package/dist/src/retention.d.ts +98 -7
- package/dist/src/retention.js +1066 -62
- package/dist/src/rotate.js +3 -3
- package/dist/src/search.d.ts +566 -8
- package/dist/src/search.js +5897 -290
- package/dist/src/secrets.d.ts +51 -7
- package/dist/src/secrets.js +316 -24
- package/dist/src/service.d.ts +49 -11
- package/dist/src/service.js +776 -35
- package/dist/src/share.js +3 -3
- package/dist/src/shared-spaces.d.ts +98 -0
- package/dist/src/shared-spaces.js +214 -0
- package/dist/src/source-reader.d.ts +73 -0
- package/dist/src/source-reader.js +715 -0
- package/dist/src/spool.d.ts +1 -1
- package/dist/src/spool.js +1 -1
- package/dist/src/start-tui.js +2 -1
- package/dist/src/start.js +2 -2
- package/dist/src/storage-endpoint.d.ts +21 -0
- package/dist/src/storage-endpoint.js +122 -0
- package/dist/src/storage-setup.js +12 -12
- package/dist/src/storage-targets.d.ts +109 -6
- package/dist/src/storage-targets.js +975 -67
- package/dist/src/stream-to-cloud.d.ts +5 -1
- package/dist/src/stream-to-cloud.js +34 -14
- package/dist/src/sync-rules.d.ts +31 -6
- package/dist/src/sync-rules.js +153 -14
- package/dist/src/team-backfill-scheduling.d.ts +8 -0
- package/dist/src/team-backfill-scheduling.js +33 -0
- package/dist/src/team-backfill.d.ts +116 -0
- package/dist/src/team-backfill.js +1429 -0
- package/dist/src/team-index-cache.d.ts +16 -0
- package/dist/src/team-index-cache.js +152 -0
- package/dist/src/team-offboarding.d.ts +38 -0
- package/dist/src/team-offboarding.js +1043 -0
- package/dist/src/team-presence.d.ts +127 -0
- package/dist/src/team-presence.js +904 -0
- package/dist/src/team-publication-policy.d.ts +20 -0
- package/dist/src/team-publication-policy.js +140 -0
- package/dist/src/team-realtime.d.ts +68 -0
- package/dist/src/team-realtime.js +816 -0
- package/dist/src/team-source-facts-cache.d.ts +23 -0
- package/dist/src/team-source-facts-cache.js +255 -0
- package/dist/src/trash.d.ts +1 -1
- package/dist/src/trash.js +2 -2
- package/dist/src/tui.js +11 -12
- package/dist/src/types.d.ts +173 -7
- package/dist/src/types.js +20 -0
- package/dist/src/ui-server.d.ts +163 -35
- package/dist/src/ui-server.js +712 -72
- package/dist/src/ui.d.ts +1 -2
- package/dist/src/ui.js +1 -2
- package/dist/src/upload.d.ts +27 -0
- package/dist/src/upload.js +383 -43
- package/dist/src/vault.d.ts +226 -30
- package/dist/src/vault.js +1776 -192
- package/dist/src/watcher.d.ts +7 -1
- package/dist/src/watcher.js +198 -55
- package/dist/src/worker.d.ts +27 -3
- package/dist/src/worker.js +274 -55
- package/package.json +33 -12
- package/scripts/native-reboot-rehearsal.mjs +90 -0
- package/web/app.js +6032 -343
- package/web/bootstrap.js +17 -0
- package/web/index.html +255 -57
- package/web/rail.js +317 -40
- package/web/retention.html +2 -2
- package/web/rules-view.js +188 -16
- package/web/sessions-view.js +485 -62
- package/web/sessions.html +2 -2
- package/web/setup-api.js +152 -29
- package/web/setup-logic.js +68 -9
- package/web/setup.html +113 -44
- package/web/setup.js +604 -71
- package/web/style.css +513 -98
- package/dist/packages/vaultline-crypto/src/chunk-access.js +0 -93
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/aead.d.ts +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/aead.js +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/format.js +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/recipients.d.ts +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/sha256-stream.d.ts +0 -0
- /package/dist/packages/{vaultline-crypto → sealkeep-crypto}/src/sha256-stream.js +0 -0
package/dist/src/retention.js
CHANGED
|
@@ -1,11 +1,25 @@
|
|
|
1
|
-
import { readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
1
|
+
import { readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { homedir } from "node:os";
|
|
2
4
|
import { join } from "node:path";
|
|
3
|
-
import { isSourceActive } from "./activity.js";
|
|
4
|
-
import { fail
|
|
5
|
+
import { isSourceActive, probeLiveNativeSession } from "./activity.js";
|
|
6
|
+
import { fail } from "./errors.js";
|
|
5
7
|
import { recordAudit } from "./audit.js";
|
|
6
|
-
import { moveToTrash } from "./trash.js";
|
|
8
|
+
import { defaultStrategy, moveToTrash } from "./trash.js";
|
|
7
9
|
import { isV2 } from "./types.js";
|
|
8
|
-
import { listArchives, readConfig,
|
|
10
|
+
import { deltaOf, listArchives, mutateArchiveRecord, readConfig, resolveDeltaChain, teamSpaceOf, updateConfigFields } from "./vault.js";
|
|
11
|
+
import { archiveCopies, hasCompleteVerifiedCopyChain, hasVerifiedCopy, verifiedCopyDestinationKey } from "./archive-copies.js";
|
|
12
|
+
import { equalHex } from "./crypto.js";
|
|
13
|
+
import { hashFileRange } from "../packages/sealkeep-crypto/src/index.js";
|
|
14
|
+
import { discardCodexResumeStub, assertCodexResumePointerMetadataSupported, assertCodexResumePointerSupported, assertCodexResumePointerTempPath, codexSessionIdMatchesFilename, installCodexResumeStub, isNativeCodexSessionPath, prepareCodexResumeStub, readUntouchedCodexResumeStub, validateNativeCodexSessionPath, validateCodexResumePointerBinding, } from "./resume-stub.js";
|
|
15
|
+
import { acquireReclaimSourceLock, beginReclaimTransaction, finalizeReclaimTransaction, installReclaimReplacement, markReclaimMetadataCommitted, markReclaimPurgeAuthorized, moveReclaimSource, reconcileReclaimTransactions, reclaimJournalPath, rollbackReclaimTransaction, validateNoFollowPath, } from "./reclaim-transaction.js";
|
|
16
|
+
import { NATIVE_ROOTS } from "./restore.js";
|
|
17
|
+
import { createBackgroundCpuYield } from "./background-cpu.js";
|
|
18
|
+
import { paceBackgroundByteStream } from "./background-bandwidth.js";
|
|
19
|
+
import { listIndexPublicationRequirements } from "./index-publication-state.js";
|
|
20
|
+
import { recheckManagedAccountIndexPublicationBeforeUnlink, verifyManagedAccountIndexPublication, } from "./index-publication-verifier.js";
|
|
21
|
+
import { resolveRecoveryPhrase } from "./secrets.js";
|
|
22
|
+
import { INDEX_EXTRACTION_POLICY } from "./search.js";
|
|
9
23
|
/** For blocker text a person reads. Whole numbers below 10 units, one decimal above. */
|
|
10
24
|
function humanBytes(value) {
|
|
11
25
|
const units = ["B", "KB", "MB", "GB", "TB"];
|
|
@@ -17,10 +31,237 @@ function humanBytes(value) {
|
|
|
17
31
|
return `${unit === 0 ? size : size < 10 ? size.toFixed(1) : Math.round(size)} ${units[unit]}`;
|
|
18
32
|
}
|
|
19
33
|
export const DEFAULT_RETENTION = {
|
|
20
|
-
|
|
34
|
+
// 30 days here meant a fresh install returned nothing for a month, which
|
|
35
|
+
// reads as "the product does not work" rather than "the product is careful".
|
|
36
|
+
// 7 matches the grace period, and the guards that actually prevent data loss
|
|
37
|
+
// — a verified remote copy, search coverage, an idle source unchanged since
|
|
38
|
+
// sealing, and the crash-safe purge transaction — are untouched by this number.
|
|
39
|
+
policy: "local-only", olderThanDays: 7, graceDays: 7, deleteAfterDays: null,
|
|
21
40
|
sourceIdleDays: null, minSourceBytes: null
|
|
22
41
|
};
|
|
23
42
|
export const POLICIES = ["local-only", "sync-only", "archive-and-reclaim", "manual-approval"];
|
|
43
|
+
const denyReclaimAuthorization = async () => false;
|
|
44
|
+
/** Current persisted policy, re-read at every irreversible boundary. */
|
|
45
|
+
function confirmedRetentionAuthorization(dataDir, delegated) {
|
|
46
|
+
return async (archiveId) => {
|
|
47
|
+
const policy = retentionSettings(await readConfig(dataDir)).policy;
|
|
48
|
+
const policyAllows = policy === "archive-and-reclaim"
|
|
49
|
+
|| (policy === "manual-approval" && (await readApprovals(dataDir)).includes(archiveId));
|
|
50
|
+
if (!policyAllows)
|
|
51
|
+
return false;
|
|
52
|
+
return delegated ? await delegated(archiveId) === true : true;
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
async function assertReclaimAuthorized(authorize, archiveId) {
|
|
56
|
+
if (!authorize)
|
|
57
|
+
return;
|
|
58
|
+
if (await authorize(archiveId) !== true) {
|
|
59
|
+
fail("invalid_argument", "Automatic reclaim was disabled or its policy changed while reclaim was running, so the original was kept.", { archiveId });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/** A delta session is durable only when its complete chain shares one target. */
|
|
63
|
+
function completeRemoteChain(archives, record) {
|
|
64
|
+
try {
|
|
65
|
+
const chain = resolveDeltaChain(archives, record);
|
|
66
|
+
return {
|
|
67
|
+
chain,
|
|
68
|
+
complete: chain.every(isV2) && hasCompleteVerifiedCopyChain(chain),
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
return { chain: [], complete: false };
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Read every object needed for this session from one complete recorded
|
|
77
|
+
* destination and require its current bytes to match the archive SHA-256. This
|
|
78
|
+
* fresh cryptographic proof closes lifecycle deletion, same-size replacement,
|
|
79
|
+
* and stale managed-metadata windows immediately before local purge.
|
|
80
|
+
*
|
|
81
|
+
* Extra stale copies do not block a healthy one. For a delta chain, however,
|
|
82
|
+
* every link must answer from the same physical destination; independently
|
|
83
|
+
* healthy links split across providers cannot restore the session.
|
|
84
|
+
*/
|
|
85
|
+
async function verifyRemoteChainImmediatelyBeforePurge(dataDir, chain, operation, options = {}) {
|
|
86
|
+
if (chain.length === 0 || !chain.every(isV2)) {
|
|
87
|
+
fail("invalid_argument", "The complete verified archive chain could not be loaded, so the local original was kept.");
|
|
88
|
+
}
|
|
89
|
+
const records = chain;
|
|
90
|
+
const verified = records.map((record) => archiveCopies(record).filter((copy) => Number.isFinite(Date.parse(copy.verifiedAt))));
|
|
91
|
+
const groups = [];
|
|
92
|
+
if (records.length === 1) {
|
|
93
|
+
for (const copy of verified[0] ?? [])
|
|
94
|
+
groups.push([copy]);
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
let common = new Set((verified[0] ?? []).map(verifiedCopyDestinationKey).filter((key) => Boolean(key)));
|
|
98
|
+
for (const copies of verified.slice(1)) {
|
|
99
|
+
const keys = new Set(copies.map(verifiedCopyDestinationKey).filter((key) => Boolean(key)));
|
|
100
|
+
common = new Set([...common].filter((key) => keys.has(key)));
|
|
101
|
+
}
|
|
102
|
+
for (const key of [...common].sort()) {
|
|
103
|
+
const group = verified.map((copies) => copies.find((copy) => verifiedCopyDestinationKey(copy) === key));
|
|
104
|
+
if (group.every((copy) => Boolean(copy)))
|
|
105
|
+
groups.push(group);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
if (groups.length === 0) {
|
|
109
|
+
fail("invalid_argument", "No one verified storage destination contains this session's complete archive chain, so the local original was kept.");
|
|
110
|
+
}
|
|
111
|
+
const check = operation ?? (async (root, archiveId, copyId) => {
|
|
112
|
+
const current = records.find((record) => record.id === archiveId);
|
|
113
|
+
const copy = archiveCopies(current).find((candidate) => candidate.copyId === copyId);
|
|
114
|
+
if (!copy)
|
|
115
|
+
return { ok: false, detail: "the selected physical copy disappeared from archive metadata" };
|
|
116
|
+
const remote = options.fetchRemoteCopy
|
|
117
|
+
? await options.fetchRemoteCopy(root, current, copy, options.signal)
|
|
118
|
+
: await import("./offload.js").then(({ fetchCiphertextStream }) => (fetchCiphertextStream(root, current, undefined, copy, {}, options.signal)));
|
|
119
|
+
const stream = options.bytesPerSecond
|
|
120
|
+
? paceBackgroundByteStream(root, remote, options.bytesPerSecond, { signal: options.signal })
|
|
121
|
+
: remote;
|
|
122
|
+
const hash = createHash("sha256");
|
|
123
|
+
let bytes = 0;
|
|
124
|
+
let sinceYield = 0;
|
|
125
|
+
for await (const chunk of stream) {
|
|
126
|
+
options.signal?.throwIfAborted();
|
|
127
|
+
hash.update(chunk);
|
|
128
|
+
bytes += chunk.length;
|
|
129
|
+
sinceYield += chunk.length;
|
|
130
|
+
if (sinceYield >= 16 * 1024 * 1024) {
|
|
131
|
+
sinceYield = 0;
|
|
132
|
+
await options.cooperate?.();
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
const digest = hash.digest("hex");
|
|
136
|
+
const expected = isV2(current) ? current.cipher : null;
|
|
137
|
+
const ok = Boolean(expected && bytes === expected.storedBytes && equalHex(digest, expected.ciphertextSha256));
|
|
138
|
+
return {
|
|
139
|
+
ok,
|
|
140
|
+
detail: ok
|
|
141
|
+
? "the complete stored bytes were fetched and matched the archive SHA-256"
|
|
142
|
+
: `the fetched object was ${bytes} bytes with SHA-256 ${digest}; it did not match this archive`,
|
|
143
|
+
};
|
|
144
|
+
});
|
|
145
|
+
const problems = [];
|
|
146
|
+
for (const group of groups) {
|
|
147
|
+
let complete = true;
|
|
148
|
+
for (let index = 0; index < records.length; index += 1) {
|
|
149
|
+
const copy = group[index];
|
|
150
|
+
try {
|
|
151
|
+
const proof = await check(dataDir, records[index].id, copy.copyId);
|
|
152
|
+
if (!proof.ok) {
|
|
153
|
+
complete = false;
|
|
154
|
+
problems.push(proof.detail);
|
|
155
|
+
break;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
complete = false;
|
|
160
|
+
problems.push(error instanceof Error ? error.message : "the provider check failed");
|
|
161
|
+
break;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (complete)
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
fail("invalid_argument", `Storage could not confirm a complete exact copy immediately before reclaim, so the local original was restored. ${problems[0] ?? "Try the copy check again."}`);
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Proves that the pathname still contains exactly the bytes the archive sealed.
|
|
171
|
+
*
|
|
172
|
+
* Size is a cheap preview gate, but it is not an integrity proof: an editor can
|
|
173
|
+
* rewrite a transcript in place without changing its length. Every destructive
|
|
174
|
+
* path calls this immediately before staging the source for reclaim, after all
|
|
175
|
+
* slower policy, remote-copy, indexing, and activity checks have finished.
|
|
176
|
+
*/
|
|
177
|
+
async function verifySourceImmediatelyBeforeReclaim(record, cooperate) {
|
|
178
|
+
const measured = await hashFileRange(record.source.path, 0, undefined, undefined, undefined, cooperate).catch(() => null);
|
|
179
|
+
if (!measured) {
|
|
180
|
+
fail("source_unreadable", "The source file could not be read at the final safety check, so nothing was moved.", { path: record.source.path });
|
|
181
|
+
}
|
|
182
|
+
if (measured.bytes !== record.source.bytes || !equalHex(measured.sha256, record.source.sha256)) {
|
|
183
|
+
fail("invalid_argument", `The session contents changed since this archive was sealed (${record.source.bytes} bytes archived, ${measured.bytes} now) — re-archive it first; nothing was moved.`, { archiveId: record.id });
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
async function verifyIndexChainImmediatelyBeforePurge(dataDir, chain, requestedHead, searchableHeads, phrase, signal, bytesPerSecond, operation) {
|
|
187
|
+
const config = await readConfig(dataDir);
|
|
188
|
+
try {
|
|
189
|
+
return await (operation ?? (async (root, links, head, unlock, currentSignal, logicalHeads = [head]) => verifyManagedAccountIndexPublication({
|
|
190
|
+
dataDir: root,
|
|
191
|
+
vaultId: config.vaultId,
|
|
192
|
+
phrase: unlock,
|
|
193
|
+
extractionPolicy: INDEX_EXTRACTION_POLICY,
|
|
194
|
+
chain: links.map((record) => ({ archiveId: record.id, archiveRef: record.id })),
|
|
195
|
+
requestedHead: { archiveId: head.id, archiveRef: head.id },
|
|
196
|
+
searchableHeads: logicalHeads.map((record) => ({ archiveId: record.id, archiveRef: record.id })),
|
|
197
|
+
signal: currentSignal,
|
|
198
|
+
maxBytesPerSecond: bytesPerSecond,
|
|
199
|
+
})))(dataDir, chain, requestedHead, phrase, signal, searchableHeads);
|
|
200
|
+
}
|
|
201
|
+
catch (error) {
|
|
202
|
+
fail("invalid_argument", `Sealkeep could not prove this exact searchable memory is in the current immutable cloud index, so the local original was kept. ${error instanceof Error ? error.message : "Index verification failed."}`, { archiveId: requestedHead.id });
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
const CODEX_SESSION_UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
206
|
+
/**
|
|
207
|
+
* A resumed Codex rollout can be reclaimed repeatedly. Each cycle creates a
|
|
208
|
+
* new archive head while every cycle keeps the same native session UUID. The
|
|
209
|
+
* next pointer searches by that UUID, so deletion is safe only if the current
|
|
210
|
+
* immutable account index still carries every previously reclaimed head.
|
|
211
|
+
*/
|
|
212
|
+
function searchableCodexContinuationHeads(records, requestedHead) {
|
|
213
|
+
if (!isV2(requestedHead) || requestedHead.source.agent !== "codex")
|
|
214
|
+
return [requestedHead];
|
|
215
|
+
const sessionId = requestedHead.reclaimed?.resumeStub?.sessionId;
|
|
216
|
+
if (!sessionId)
|
|
217
|
+
return [requestedHead];
|
|
218
|
+
if (!CODEX_SESSION_UUID.test(sessionId)
|
|
219
|
+
|| !codexSessionIdMatchesFilename(requestedHead.source.path, sessionId)) {
|
|
220
|
+
fail("invalid_argument", "The current Codex continuation is not bound to its exact native session id, so the local original was kept.", { archiveId: requestedHead.id });
|
|
221
|
+
}
|
|
222
|
+
const heads = records.filter((candidate) => isV2(candidate)
|
|
223
|
+
&& candidate.reclaimed?.resumeStub?.sessionId?.toLowerCase() === sessionId.toLowerCase());
|
|
224
|
+
for (const candidate of heads) {
|
|
225
|
+
if (candidate.source.agent !== "codex"
|
|
226
|
+
|| !codexSessionIdMatchesFilename(candidate.source.path, sessionId)) {
|
|
227
|
+
fail("invalid_argument", `A prior reclaimed continuation (${candidate.id}) is not bound to the same exact Codex session id, so the local original was kept.`, { archiveId: requestedHead.id, priorArchiveId: candidate.id });
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
if (!heads.some((candidate) => candidate.id === requestedHead.id)) {
|
|
231
|
+
fail("invalid_argument", "The current Codex continuation reclaim record changed before index verification, so the local original was kept.", { archiveId: requestedHead.id });
|
|
232
|
+
}
|
|
233
|
+
return heads.sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id));
|
|
234
|
+
}
|
|
235
|
+
async function verifyCurrentDurabilityImmediatelyBeforePurge(dataDir, archiveId, sourcePath, remoteCheck, activity = {}, options = {}) {
|
|
236
|
+
const records = await listArchives(dataDir);
|
|
237
|
+
const record = records.find((candidate) => candidate.id === archiveId);
|
|
238
|
+
if (!record)
|
|
239
|
+
throw new Error("Reclaim archive disappeared before final durability verification");
|
|
240
|
+
const remoteChain = completeRemoteChain(records, record);
|
|
241
|
+
if (!remoteChain.complete)
|
|
242
|
+
throw new Error("The complete verified archive chain changed before purge");
|
|
243
|
+
// The byte-for-byte provider read is deliberately first: it is the slowest
|
|
244
|
+
// gate. Mutable local decisions (search coverage, a newly added pin, and a
|
|
245
|
+
// waiting agent process) are sampled after it, adjacent to authorization.
|
|
246
|
+
await verifyRemoteChainImmediatelyBeforePurge(dataDir, remoteChain.chain, remoteCheck, options);
|
|
247
|
+
const config = await readConfig(dataDir);
|
|
248
|
+
const phrase = await resolveRecoveryPhrase(dataDir, config.vaultId, options.phrase).catch(() => null);
|
|
249
|
+
if (!phrase) {
|
|
250
|
+
fail("invalid_argument", "Sealkeep cannot verify searchable cloud memory while this machine is locked. Unlock it and try again; the local original was kept.");
|
|
251
|
+
}
|
|
252
|
+
const publication = await verifyIndexChainImmediatelyBeforePurge(dataDir, remoteChain.chain, record, searchableCodexContinuationHeads(records, record), phrase, options.signal, options.bytesPerSecond, options.verifyIndexPublication);
|
|
253
|
+
if (isPinned(sourcePath, await neverReclaimMarks(dataDir))) {
|
|
254
|
+
fail("invalid_argument", "This session was pinned while reclaim was running, so the original was restored.", { archiveId });
|
|
255
|
+
}
|
|
256
|
+
const live = await (activity.liveNativeSession ?? probeLiveNativeSession)(sourcePath);
|
|
257
|
+
if (live === null) {
|
|
258
|
+
fail("invalid_argument", "Sealkeep could not confirm that the native agent process is stopped, so the original was restored.", { archiveId });
|
|
259
|
+
}
|
|
260
|
+
if (live) {
|
|
261
|
+
fail("invalid_argument", "This session belongs to a running native agent process, so the original was restored.", { archiveId });
|
|
262
|
+
}
|
|
263
|
+
return publication;
|
|
264
|
+
}
|
|
24
265
|
export function retentionSettings(config, overrides = {}) {
|
|
25
266
|
return { ...DEFAULT_RETENTION, ...config.retention, ...overrides };
|
|
26
267
|
}
|
|
@@ -44,6 +285,73 @@ export async function approveReclamation(dataDir, archiveIds) {
|
|
|
44
285
|
await recordAudit(dataDir, "retention.approve", "allowed", { approved: archiveIds.length, total: merged.length });
|
|
45
286
|
return merged;
|
|
46
287
|
}
|
|
288
|
+
/**
|
|
289
|
+
* The standing "never take this one" list, honoured by EVERY reclaimer.
|
|
290
|
+
*
|
|
291
|
+
* `<dataDir>/never-reclaim` (one id or path fragment per line, `#` comments)
|
|
292
|
+
* was read by exactly one of the four deleters — the squeeze command — so a
|
|
293
|
+
* pinned session survived a squeeze and was then reclaimed by the daemon's
|
|
294
|
+
* ordinary retention tick, with no user in the loop. A pin only one deleter
|
|
295
|
+
* honours is a hole; it lives here now so every caller inherits it.
|
|
296
|
+
*/
|
|
297
|
+
export async function neverReclaimMarks(dataDir) {
|
|
298
|
+
const { readFile } = await import("node:fs/promises");
|
|
299
|
+
const { join } = await import("node:path");
|
|
300
|
+
return readFile(join(dataDir, "never-reclaim"), "utf8")
|
|
301
|
+
.then((text) => text.split("\n").map((line) => line.split("#")[0].trim().toLowerCase()).filter(Boolean))
|
|
302
|
+
.catch(() => []);
|
|
303
|
+
}
|
|
304
|
+
export function isPinned(path, marks) {
|
|
305
|
+
const lower = path.toLowerCase();
|
|
306
|
+
return marks.some((mark) => lower.includes(mark));
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Which local archive ids the encrypted content index can currently answer.
|
|
310
|
+
*
|
|
311
|
+
* A verified remote object proves recoverability, but preservation also means
|
|
312
|
+
* an agent can find the work after the bulky source leaves this machine. The
|
|
313
|
+
* coverage sidecar contains ids only (never transcript text), so this gate is
|
|
314
|
+
* cheap and does not need the recovery phrase. `null` is deliberately fail
|
|
315
|
+
* closed: if coverage cannot be read, no source becomes disposable merely
|
|
316
|
+
* because an advisory file is damaged.
|
|
317
|
+
*/
|
|
318
|
+
async function indexedArchiveIds(dataDir, records) {
|
|
319
|
+
try {
|
|
320
|
+
const { indexCoverage } = await import("./search.js");
|
|
321
|
+
const coverage = await indexCoverage(dataDir);
|
|
322
|
+
const missing = new Set(coverage.missing.map((item) => item.id));
|
|
323
|
+
return new Set(records.map((record) => record.id).filter((id) => !missing.has(id)));
|
|
324
|
+
}
|
|
325
|
+
catch {
|
|
326
|
+
return null;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* Cheap, local preview of the durable publication ledger. Local index
|
|
331
|
+
* coverage says an agent on THIS machine can search; a positive receipt says
|
|
332
|
+
* an immutable managed generation carrying that exact entry was committed and
|
|
333
|
+
* read back. Destructive paths repeat the network and cryptographic checks at
|
|
334
|
+
* the final boundary; this merely prevents the UI/daemon from presenting an
|
|
335
|
+
* obviously unreceipted source as reclaimable.
|
|
336
|
+
*/
|
|
337
|
+
async function publicationReceiptStatuses(dataDir, config) {
|
|
338
|
+
try {
|
|
339
|
+
return await listIndexPublicationRequirements(dataDir, { vaultId: config.vaultId });
|
|
340
|
+
}
|
|
341
|
+
catch {
|
|
342
|
+
return null;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
function hasPublicationReceipt(statuses, archiveId, scope) {
|
|
346
|
+
if (!statuses)
|
|
347
|
+
return false;
|
|
348
|
+
return statuses.some((status) => status.requirement.archiveId === archiveId
|
|
349
|
+
&& status.receipt !== null
|
|
350
|
+
&& status.requirement.scope.kind === scope.kind
|
|
351
|
+
&& (scope.kind === "account"
|
|
352
|
+
? status.requirement.scope.kind === "account" && status.requirement.scope.vaultId === scope.vaultId
|
|
353
|
+
: status.requirement.scope.kind === "team" && status.requirement.scope.spaceKey === scope.spaceKey));
|
|
354
|
+
}
|
|
47
355
|
/**
|
|
48
356
|
* Decides, per archive, whether its local source may be reclaimed.
|
|
49
357
|
*
|
|
@@ -51,6 +359,7 @@ export async function approveReclamation(dataDir, archiveIds) {
|
|
|
51
359
|
* complete reason a file is protected rather than the first one that matched.
|
|
52
360
|
*/
|
|
53
361
|
export async function evaluateRetention(dataDir, options = {}) {
|
|
362
|
+
const pins = await neverReclaimMarks(dataDir);
|
|
54
363
|
const config = await readConfig(dataDir);
|
|
55
364
|
const settings = retentionSettings(config, { ...options.settings, ...(options.olderThanDays === undefined ? {} : { olderThanDays: options.olderThanDays }) });
|
|
56
365
|
if (!Number.isFinite(settings.olderThanDays) || settings.olderThanDays < 0)
|
|
@@ -67,7 +376,9 @@ export async function evaluateRetention(dataDir, options = {}) {
|
|
|
67
376
|
// managed storage proves it, and a fresh vault with none reads as unmanaged,
|
|
68
377
|
// which only changes wording.
|
|
69
378
|
const allRecords = await listArchives(dataDir);
|
|
70
|
-
const
|
|
379
|
+
const indexed = await indexedArchiveIds(dataDir, allRecords);
|
|
380
|
+
const publicationStatuses = await publicationReceiptStatuses(dataDir, config);
|
|
381
|
+
const managed = allRecords.some((item) => isV2(item) && archiveCopies(item).some((copy) => copy.provider === "vaultline" && copy.verifiedAt));
|
|
71
382
|
for (const record of allRecords) {
|
|
72
383
|
const current = await stat(record.source.path).catch(() => null);
|
|
73
384
|
if (!current)
|
|
@@ -75,6 +386,8 @@ export async function evaluateRetention(dataDir, options = {}) {
|
|
|
75
386
|
if (isV2(record) && record.reclaimed)
|
|
76
387
|
continue;
|
|
77
388
|
const blockers = [];
|
|
389
|
+
if (isPinned(record.source.path, pins))
|
|
390
|
+
blockers.push("listed in never-reclaim — this session is pinned; edit <dataDir>/never-reclaim to release it");
|
|
78
391
|
// The archive covers the bytes that existed when it was written. A session
|
|
79
392
|
// that kept being appended to has content in no archive at all, and
|
|
80
393
|
// reclaiming it would delete exactly that content — so refuse, and say so
|
|
@@ -82,6 +395,8 @@ export async function evaluateRetention(dataDir, options = {}) {
|
|
|
82
395
|
// agent transcripts grow by appending and this runs over every record.
|
|
83
396
|
if (current.size !== record.source.bytes)
|
|
84
397
|
blockers.push(`source changed since it was archived (${record.source.bytes} bytes archived, ${current.size} now); re-archive it first`);
|
|
398
|
+
if (current.nlink !== 1)
|
|
399
|
+
blockers.push(`source has ${current.nlink} hard links, so removing this name would not free its allocated blocks`);
|
|
85
400
|
// These render straight into a dashboard table cell, so they are sentences
|
|
86
401
|
// a customer reads, not slugs a maintainer greps. The retention page already
|
|
87
402
|
// names the four policies in plain words; the same words are used here.
|
|
@@ -90,28 +405,44 @@ export async function evaluateRetention(dataDir, options = {}) {
|
|
|
90
405
|
// shows in a narrow column. The one a person can act on leads — "no
|
|
91
406
|
// storage is set up" is a thing to go and change, while "set to keep
|
|
92
407
|
// everything" is quite possibly what they meant.
|
|
93
|
-
const remote = isV2(record) ? record.
|
|
408
|
+
const remote = isV2(record) ? archiveCopies(record).find((copy) => copy.verifiedAt) : undefined;
|
|
409
|
+
const remoteChain = isV2(record) ? completeRemoteChain(allRecords, record) : { chain: [], complete: false };
|
|
94
410
|
// A managed-tier vault has no `remoteStorage` of its own — the bucket is
|
|
95
411
|
// ours — so "no remote target configured" was told to paying customers about
|
|
96
412
|
// storage they were paying for. The honest state is the same either way:
|
|
97
413
|
// this archive has not been uploaded and verified yet.
|
|
98
414
|
if (!remote?.verifiedAt)
|
|
99
415
|
blockers.push(config.remoteStorage || managed ? "not yet uploaded and verified" : "no storage is set up, so there is no second copy to reclaim against");
|
|
416
|
+
else if (!remoteChain.complete && !deltaOf(record))
|
|
417
|
+
blockers.push("the archive chain is not fully verified at one storage destination yet");
|
|
418
|
+
// Durability and discoverability are separate promises. Upload may run in
|
|
419
|
+
// parallel with a historical index migration, but the original transcript
|
|
420
|
+
// stays until this exact archive is searchable. That keeps the fast lane
|
|
421
|
+
// fast without ever turning a successful backup into forgotten work.
|
|
422
|
+
if (!indexed?.has(record.id))
|
|
423
|
+
blockers.push(indexed ? "not yet searchable; indexing is still in progress" : "search index could not be checked; keeping the source");
|
|
424
|
+
const accountReceipted = remoteChain.chain.length > 0
|
|
425
|
+
&& remoteChain.chain.every((link) => hasPublicationReceipt(publicationStatuses, link.id, { kind: "account", vaultId: config.vaultId }));
|
|
426
|
+
if (!accountReceipted)
|
|
427
|
+
blockers.push(publicationStatuses
|
|
428
|
+
? "search memory is not yet committed to a verified Sealkeep Cloud index generation"
|
|
429
|
+
: "search publication proof could not be checked; keeping the source");
|
|
430
|
+
const teamBinding = teamSpaceOf(config, isV2(record) ? record.source.project : undefined, isV2(record) ? record.source.projectKey : undefined);
|
|
431
|
+
if (teamBinding && !remoteChain.chain.every((link) => hasPublicationReceipt(publicationStatuses, link.id, { kind: "team", spaceKey: teamBinding.spaceKey }))) {
|
|
432
|
+
blockers.push("shared project memory has not reached its current verified team index yet");
|
|
433
|
+
}
|
|
100
434
|
// A delta archive restores through its whole chain, so "there is a second
|
|
101
435
|
// copy" must be true of every link, not just this head. One unverified
|
|
102
436
|
// base and reclaiming the source deletes bytes only this disk holds.
|
|
103
|
-
if (remote?.verifiedAt) {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
catch {
|
|
113
|
-
blockers.push("this snapshot builds on an earlier archive that is missing, so the transcript cannot be restored from the cloud alone");
|
|
114
|
-
}
|
|
437
|
+
if (remote?.verifiedAt && deltaOf(record) && !remoteChain.complete) {
|
|
438
|
+
if (remoteChain.chain.length > 0) {
|
|
439
|
+
const unverified = remoteChain.chain.filter((link) => !(isV2(link) && hasVerifiedCopy(link))).length;
|
|
440
|
+
blockers.push(unverified > 0
|
|
441
|
+
? `this snapshot builds on ${remoteChain.chain.length - 1} earlier ${remoteChain.chain.length === 2 ? "archive" : "archives"}, and ${unverified} of the chain ${unverified === 1 ? "is" : "are"} not uploaded and verified at one destination yet`
|
|
442
|
+
: "this snapshot's verified copies are split across storage destinations, so no one destination can restore the cloud session");
|
|
443
|
+
}
|
|
444
|
+
else {
|
|
445
|
+
blockers.push("this snapshot builds on an earlier archive that is missing, so the cloud cannot reproduce the file. Keep the source.");
|
|
115
446
|
}
|
|
116
447
|
}
|
|
117
448
|
if (settings.policy === "local-only")
|
|
@@ -142,6 +473,15 @@ export async function evaluateRetention(dataDir, options = {}) {
|
|
|
142
473
|
blockers.push(hours < 48 ? `grace period ends in ${hours}h` : `grace period ends in ${Math.ceil(hours / 24)} days`);
|
|
143
474
|
}
|
|
144
475
|
}
|
|
476
|
+
// A multi-agent Codex child currently cannot resume independently through
|
|
477
|
+
// its own native id. Detect that before presenting it as reclaimable; the
|
|
478
|
+
// apply path repeats this parser while building the pointer and therefore
|
|
479
|
+
// still fails closed if the file changes after preview.
|
|
480
|
+
if (blockers.length === 0 && record.source.agent === "codex"
|
|
481
|
+
&& await isNativeCodexSessionPath(record.source.path, options.home ?? homedir())) {
|
|
482
|
+
await assertCodexResumePointerSupported(record.source.path, options.home ?? homedir())
|
|
483
|
+
.catch((error) => blockers.push(error instanceof Error ? error.message : "Codex resume metadata could not be verified; keeping the original on disk"));
|
|
484
|
+
}
|
|
145
485
|
// The activity probe shells out to lsof, so it is the one expensive gate —
|
|
146
486
|
// and it only changes the outcome when every other gate already passed.
|
|
147
487
|
// Probing unconditionally put ~90ms × every archive inside each evaluation,
|
|
@@ -159,12 +499,14 @@ export async function evaluateRetention(dataDir, options = {}) {
|
|
|
159
499
|
// reason.
|
|
160
500
|
const distinct = [...new Set(blockers)];
|
|
161
501
|
candidates.push({
|
|
162
|
-
id: record.id, agent: record.source.agent, path: record.source.path, bytes: record.source.bytes,
|
|
502
|
+
id: record.id, agent: record.source.agent, path: record.source.path, bytes: record.source.bytes,
|
|
503
|
+
allocatedBytes: current.blocks * 512, createdAt: record.createdAt,
|
|
163
504
|
eligible: distinct.length === 0, blockers: distinct, reason: distinct[0] ?? "eligible for reclamation",
|
|
164
505
|
remote: remote?.verifiedAt ? { provider: remote.provider, verifiedAt: remote.verifiedAt } : null
|
|
165
506
|
});
|
|
166
507
|
}
|
|
167
|
-
const reclaimableBytes = candidates.filter((candidate) => candidate.eligible)
|
|
508
|
+
const reclaimableBytes = candidates.filter((candidate) => candidate.eligible)
|
|
509
|
+
.reduce((total, candidate) => total + candidate.allocatedBytes, 0);
|
|
168
510
|
return {
|
|
169
511
|
mode: "preview", policy: settings.policy, olderThanDays: settings.olderThanDays, graceDays: settings.graceDays,
|
|
170
512
|
sourceIdleDays: settings.sourceIdleDays, minSourceBytes: settings.minSourceBytes,
|
|
@@ -175,37 +517,632 @@ export async function evaluateRetention(dataDir, options = {}) {
|
|
|
175
517
|
reclaimableBytes, candidates
|
|
176
518
|
};
|
|
177
519
|
}
|
|
520
|
+
/**
|
|
521
|
+
* Stages a source and, for a real native Codex rollout, replaces it with the
|
|
522
|
+
* already-prepared resume pointer. Preparation happens while the verified
|
|
523
|
+
* original still exists. Any failure before the durable purge decision rolls
|
|
524
|
+
* the move back rather than leaving resume broken.
|
|
525
|
+
*/
|
|
526
|
+
async function moveReclaimableSource(dataDir, record, options) {
|
|
527
|
+
const stub = await prepareCodexResumeStub(record.source.path, record.id, {
|
|
528
|
+
home: options.home,
|
|
529
|
+
project: isV2(record) ? record.source.project ?? null : null,
|
|
530
|
+
});
|
|
531
|
+
const selectedHome = options.home ?? homedir();
|
|
532
|
+
const nativeClaudeSource = record.source.agent === "claude"
|
|
533
|
+
&& await validateNoFollowPath(record.source.path, NATIVE_ROOTS.claude(selectedHome), { leaf: "file" })
|
|
534
|
+
.then(() => true, () => false);
|
|
535
|
+
const durablePurge = Boolean(stub) || nativeClaudeSource;
|
|
536
|
+
// Every production "Free up disk" operation is one journaled transaction.
|
|
537
|
+
// Codex gets a tiny same-id replacement; other agents leave the native path
|
|
538
|
+
// absent. Test seams for the legacy trash/install boundary stay narrow and
|
|
539
|
+
// never run in production.
|
|
540
|
+
if (durablePurge && !options.operations?.moveToTrash && !options.operations?.installResumeStub) {
|
|
541
|
+
if (!options.lock)
|
|
542
|
+
throw new Error("Source reclaim requires its source lock");
|
|
543
|
+
if (stub) {
|
|
544
|
+
await validateNativeCodexSessionPath(record.source.path, options.home, "file");
|
|
545
|
+
await validateNativeCodexSessionPath(stub.tempPath, options.home, "file");
|
|
546
|
+
}
|
|
547
|
+
let transaction;
|
|
548
|
+
try {
|
|
549
|
+
transaction = await beginReclaimTransaction(dataDir, options.lock, {
|
|
550
|
+
sourcePath: record.source.path,
|
|
551
|
+
original: { bytes: record.source.bytes, sha256: record.source.sha256 },
|
|
552
|
+
replacement: stub ? { tempPath: stub.tempPath, bytes: stub.bytes, sha256: stub.sha256 } : null,
|
|
553
|
+
transactionId: record.id,
|
|
554
|
+
}, { onYield: options.cooperate });
|
|
555
|
+
}
|
|
556
|
+
catch (error) {
|
|
557
|
+
// `begin` publishes the durable journal last. If a directory fsync
|
|
558
|
+
// reports failure after that publication, its exact replacement temp is
|
|
559
|
+
// now recovery state and must survive for the next reconciliation. In
|
|
560
|
+
// every other begin failure this process still owns the unreferenced
|
|
561
|
+
// prepared temp and can remove it.
|
|
562
|
+
const raw = await readFile(reclaimJournalPath(dataDir, record.id), "utf8").catch(() => null);
|
|
563
|
+
let journalOwnsTemp = false;
|
|
564
|
+
if (raw && stub) {
|
|
565
|
+
try {
|
|
566
|
+
const parsed = JSON.parse(raw);
|
|
567
|
+
journalOwnsTemp = parsed.replacement?.tempPath === stub.tempPath
|
|
568
|
+
&& parsed.replacement.bytes === stub.bytes
|
|
569
|
+
&& parsed.replacement.sha256 === stub.sha256;
|
|
570
|
+
}
|
|
571
|
+
catch { /* malformed journals stay visible as reconciliation conflicts */ }
|
|
572
|
+
}
|
|
573
|
+
if (!journalOwnsTemp)
|
|
574
|
+
await discardCodexResumeStub(stub);
|
|
575
|
+
throw error;
|
|
576
|
+
}
|
|
577
|
+
try {
|
|
578
|
+
await assertReclaimAuthorized(options.authorizeReclaim, record.id);
|
|
579
|
+
await options.assertStillAllowed?.();
|
|
580
|
+
await moveReclaimSource(transaction, options.lock, { onYield: options.cooperate });
|
|
581
|
+
if (stub)
|
|
582
|
+
await installReclaimReplacement(transaction, options.lock, { onYield: options.cooperate });
|
|
583
|
+
// Re-check the parked inode after its public native name has gone. An
|
|
584
|
+
// agent that already held the old file descriptor can otherwise keep
|
|
585
|
+
// writing to it after our first lsof probe. At this point new agent opens
|
|
586
|
+
// reach either the Codex pointer or an absent Claude path; any handle on
|
|
587
|
+
// the staged inode is therefore an old live writer and forces rollback.
|
|
588
|
+
const stagedActivity = await isSourceActive(transaction.trashPath, {
|
|
589
|
+
...options.activity,
|
|
590
|
+
now: options.now ?? Date.now(),
|
|
591
|
+
});
|
|
592
|
+
if (stagedActivity.active) {
|
|
593
|
+
fail("invalid_argument", `The session became active while Sealkeep prepared to reclaim it: ${stagedActivity.reason}. The original was restored and nothing was freed.`, { archiveId: record.id });
|
|
594
|
+
}
|
|
595
|
+
return {
|
|
596
|
+
trash: { movedTo: transaction.trashPath, strategy: "sealkeep-trash" },
|
|
597
|
+
stub,
|
|
598
|
+
transaction,
|
|
599
|
+
lock: options.lock,
|
|
600
|
+
verifyPurgeStillSafe: async () => {
|
|
601
|
+
const publication = await verifyCurrentDurabilityImmediatelyBeforePurge(dataDir, record.id, record.source.path, options.operations?.checkRemoteCopy, options.activity, {
|
|
602
|
+
signal: options.signal,
|
|
603
|
+
cooperate: options.cooperate,
|
|
604
|
+
bytesPerSecond: options.remoteBytesPerSecond,
|
|
605
|
+
fetchRemoteCopy: options.operations?.fetchRemoteCopy,
|
|
606
|
+
phrase: options.phrase,
|
|
607
|
+
verifyIndexPublication: options.operations?.verifyIndexPublication,
|
|
608
|
+
});
|
|
609
|
+
const finalStagedActivity = await isSourceActive(transaction.trashPath, {
|
|
610
|
+
...options.activity,
|
|
611
|
+
now: Date.now(),
|
|
612
|
+
});
|
|
613
|
+
if (finalStagedActivity.active) {
|
|
614
|
+
fail("invalid_argument", `The staged session is still open: ${finalStagedActivity.reason}. The original was restored.`, { archiveId: record.id });
|
|
615
|
+
}
|
|
616
|
+
// The full provider read above can take minutes. Sample the daemon's
|
|
617
|
+
// current permission last, adjacent to the irreversible unlink.
|
|
618
|
+
await assertReclaimAuthorized(options.authorizeReclaim, record.id);
|
|
619
|
+
await (options.operations?.recheckIndexPublication
|
|
620
|
+
? options.operations.recheckIndexPublication(dataDir, publication, options.signal)
|
|
621
|
+
: recheckManagedAccountIndexPublicationBeforeUnlink({
|
|
622
|
+
dataDir,
|
|
623
|
+
verification: publication,
|
|
624
|
+
signal: options.signal,
|
|
625
|
+
}));
|
|
626
|
+
},
|
|
627
|
+
};
|
|
628
|
+
}
|
|
629
|
+
catch (error) {
|
|
630
|
+
const rollbackErrors = [];
|
|
631
|
+
await rollbackReclaimTransaction(transaction, options.lock, { io: { onYield: options.cooperate } })
|
|
632
|
+
.catch((rollback) => rollbackErrors.push(rollback));
|
|
633
|
+
await discardCodexResumeStub(stub);
|
|
634
|
+
if (rollbackErrors.length > 0)
|
|
635
|
+
throw new AggregateError([error, ...rollbackErrors], "Codex reclamation failed and its journal was retained for reconciliation");
|
|
636
|
+
throw error;
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
// The public action is named "Free up disk". Moving an unsupported source
|
|
640
|
+
// to another directory on the same filesystem frees no blocks, so production
|
|
641
|
+
// must not silently fall back to Trash and report a successful reclaim.
|
|
642
|
+
// The injected boundary remains only for the older isolated trash-contract
|
|
643
|
+
// tests; real native Codex and Claude paths always use the journaled purge.
|
|
644
|
+
if (!options.operations?.moveToTrash && !options.operations?.installResumeStub) {
|
|
645
|
+
fail("invalid_argument", "Sealkeep cannot safely free this source yet. Only supported native Codex and Claude session files can be permanently reclaimed; the original was kept.", { archiveId: record.id, agent: record.source.agent });
|
|
646
|
+
}
|
|
647
|
+
let trash = null;
|
|
648
|
+
try {
|
|
649
|
+
// Windows' native Recycle Bin API does not return the moved pathname, so
|
|
650
|
+
// it cannot roll the original back if pointer installation fails. A Codex
|
|
651
|
+
// pointer therefore uses Sealkeep's own recoverable Trash on Windows.
|
|
652
|
+
const requestedStrategy = options.strategy ?? defaultStrategy();
|
|
653
|
+
const strategy = stub && requestedStrategy === "windows-recycle-bin" ? "sealkeep-trash" : requestedStrategy;
|
|
654
|
+
await assertReclaimAuthorized(options.authorizeReclaim, record.id);
|
|
655
|
+
trash = await (options.operations?.moveToTrash ?? moveToTrash)(record.source.path, { dataDir, home: options.home, strategy });
|
|
656
|
+
if (stub)
|
|
657
|
+
await (options.operations?.installResumeStub ?? installCodexResumeStub)(stub);
|
|
658
|
+
return { trash, stub };
|
|
659
|
+
}
|
|
660
|
+
catch (error) {
|
|
661
|
+
await discardCodexResumeStub(stub);
|
|
662
|
+
// macOS, XDG and Sealkeep-owned Trash all return a real path. Codex never
|
|
663
|
+
// reaches this branch with the pathless Windows Recycle Bin strategy.
|
|
664
|
+
if (trash && trash.strategy !== "windows-recycle-bin") {
|
|
665
|
+
const sourceMissing = !(await stat(record.source.path).catch(() => null));
|
|
666
|
+
const trashed = await stat(trash.movedTo).catch(() => null);
|
|
667
|
+
if (sourceMissing && trashed?.isFile())
|
|
668
|
+
await rename(trash.movedTo, record.source.path).catch(() => undefined);
|
|
669
|
+
}
|
|
670
|
+
throw error;
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
function resumeStubRecord(stub) {
|
|
674
|
+
return stub ? { resumeStub: { version: 1, sessionId: stub.sessionId, bytes: stub.bytes, sha256: stub.sha256 } } : {};
|
|
675
|
+
}
|
|
676
|
+
function transactionAllocatedBytesFreed(transaction) {
|
|
677
|
+
return Math.max(0, transaction.original.allocatedBytes - (transaction.replacement?.allocatedBytes ?? 0));
|
|
678
|
+
}
|
|
679
|
+
function hasPlausibleFinalFreedBytes(reclaimed, transaction) {
|
|
680
|
+
return reclaimed.stagingPath === undefined
|
|
681
|
+
&& typeof reclaimed.purgedAt === "string"
|
|
682
|
+
&& Number.isSafeInteger(reclaimed.freedBytes)
|
|
683
|
+
&& (reclaimed.freedBytes ?? -1) >= 0
|
|
684
|
+
&& (reclaimed.freedBytes ?? Number.MAX_SAFE_INTEGER) <= transactionAllocatedBytesFreed(transaction);
|
|
685
|
+
}
|
|
686
|
+
/** Put the exact original back without ever overwriting content written after the move. */
|
|
687
|
+
async function rollbackReclaimMove(record, moved) {
|
|
688
|
+
if (moved.trash.strategy === "windows-recycle-bin") {
|
|
689
|
+
throw new Error("The source entered the Windows Recycle Bin and cannot be rolled back automatically");
|
|
690
|
+
}
|
|
691
|
+
const trashed = await stat(moved.trash.movedTo).catch(() => null);
|
|
692
|
+
if (!trashed?.isFile())
|
|
693
|
+
throw new Error(`The reclaimed original is not available at ${moved.trash.movedTo}`);
|
|
694
|
+
if (!moved.stub) {
|
|
695
|
+
if (await stat(record.source.path).catch(() => null))
|
|
696
|
+
throw new Error(`Refusing to overwrite new content at ${record.source.path}`);
|
|
697
|
+
await rename(moved.trash.movedTo, record.source.path);
|
|
698
|
+
return;
|
|
699
|
+
}
|
|
700
|
+
const pointer = await readUntouchedCodexResumeStub(record.source.path);
|
|
701
|
+
if (!pointer
|
|
702
|
+
|| pointer.archiveId !== record.id
|
|
703
|
+
|| pointer.sessionId !== moved.stub.sessionId
|
|
704
|
+
|| pointer.bytes !== moved.stub.bytes
|
|
705
|
+
|| pointer.sha256 !== moved.stub.sha256) {
|
|
706
|
+
throw new Error(`The Codex resume pointer changed before rollback; refusing to overwrite ${record.source.path}`);
|
|
707
|
+
}
|
|
708
|
+
// Park the pointer with rename and restore the original. Retain the tiny
|
|
709
|
+
// parked pointer even after success: Codex may have opened its old inode
|
|
710
|
+
// before the rename and can append a resumed turn through that fd. Unlinking
|
|
711
|
+
// it here would silently discard that turn. If restoration fails, atomically
|
|
712
|
+
// put the pointer back so Codex's catalog is never left with a missing path.
|
|
713
|
+
const parked = `${moved.stub.tempPath}.rollback`;
|
|
714
|
+
await rename(record.source.path, parked);
|
|
715
|
+
try {
|
|
716
|
+
await rename(moved.trash.movedTo, record.source.path);
|
|
717
|
+
}
|
|
718
|
+
catch (error) {
|
|
719
|
+
await rename(parked, record.source.path).catch(() => undefined);
|
|
720
|
+
throw error;
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
async function commitReclaim(dataDir, storageRoot, record, moved, reclaimedAt, detail, operations = {}) {
|
|
724
|
+
const transaction = moved.transaction;
|
|
725
|
+
const reclaimed = transaction ? {
|
|
726
|
+
at: reclaimedAt,
|
|
727
|
+
strategy: "purged",
|
|
728
|
+
transactionId: transaction.id,
|
|
729
|
+
stagingPath: transaction.trashPath,
|
|
730
|
+
originalBytes: transaction.original.bytes,
|
|
731
|
+
...resumeStubRecord(moved.stub),
|
|
732
|
+
} : {
|
|
733
|
+
// Injectable legacy seams deliberately retain their recoverable Trash
|
|
734
|
+
// behavior. Real Free-up-disk calls always use the durable purge path.
|
|
735
|
+
at: reclaimedAt,
|
|
736
|
+
trashPath: moved.trash.movedTo,
|
|
737
|
+
strategy: moved.trash.strategy,
|
|
738
|
+
...resumeStubRecord(moved.stub),
|
|
739
|
+
};
|
|
740
|
+
const mutate = operations.mutateRecord ?? mutateArchiveRecord;
|
|
741
|
+
const audit = operations.audit ?? recordAudit;
|
|
742
|
+
const matchesReclaimed = (current) => current !== undefined
|
|
743
|
+
&& isV2(current)
|
|
744
|
+
&& current.reclaimed?.at === reclaimed.at
|
|
745
|
+
&& current.reclaimed.strategy === reclaimed.strategy
|
|
746
|
+
&& current.reclaimed.transactionId === reclaimed.transactionId
|
|
747
|
+
&& current.reclaimed.originalBytes === reclaimed.originalBytes
|
|
748
|
+
&& (transaction
|
|
749
|
+
? current.reclaimed.stagingPath === transaction.trashPath || (hasPlausibleFinalFreedBytes(current.reclaimed, transaction))
|
|
750
|
+
: current.reclaimed.trashPath === reclaimed.trashPath)
|
|
751
|
+
&& current.reclaimed.resumeStub?.sessionId === reclaimed.resumeStub?.sessionId
|
|
752
|
+
&& current.reclaimed.resumeStub?.bytes === reclaimed.resumeStub?.bytes
|
|
753
|
+
&& current.reclaimed.resumeStub?.sha256 === reclaimed.resumeStub?.sha256;
|
|
754
|
+
const readCurrent = async () => {
|
|
755
|
+
try {
|
|
756
|
+
return JSON.parse(await readFile(join(storageRoot, `${record.id}.json`), "utf8"));
|
|
757
|
+
}
|
|
758
|
+
catch (error) {
|
|
759
|
+
if (error.code === "ENOENT")
|
|
760
|
+
return undefined;
|
|
761
|
+
throw error;
|
|
762
|
+
}
|
|
763
|
+
};
|
|
764
|
+
try {
|
|
765
|
+
await mutate(storageRoot, record.id, (current) => isV2(current) ? { ...current, reclaimed } : current);
|
|
766
|
+
if (!transaction || !moved.lock) {
|
|
767
|
+
await audit(dataDir, "source.reclaim", "allowed", detail);
|
|
768
|
+
return { freedBytes: 0 };
|
|
769
|
+
}
|
|
770
|
+
await markReclaimMetadataCommitted(transaction, moved.lock);
|
|
771
|
+
const expectedFreedBytes = transactionAllocatedBytesFreed(transaction);
|
|
772
|
+
const auditDetail = {
|
|
773
|
+
...detail,
|
|
774
|
+
transactionId: transaction.id,
|
|
775
|
+
purgeAuthorized: true,
|
|
776
|
+
expectedFreedBytes,
|
|
777
|
+
};
|
|
778
|
+
// Audit first. The journal's purge-authorized stage is the durable proof
|
|
779
|
+
// that both record metadata and this decision reached storage.
|
|
780
|
+
await audit(dataDir, "source.reclaim", "allowed", auditDetail);
|
|
781
|
+
await markReclaimPurgeAuthorized(transaction, moved.lock);
|
|
782
|
+
if (!moved.verifyPurgeStillSafe) {
|
|
783
|
+
throw new Error("Reclaim cannot purge without a fresh provider proof at the irreversible boundary");
|
|
784
|
+
}
|
|
785
|
+
let purgedAt;
|
|
786
|
+
const finalized = await finalizeReclaimTransaction(transaction, moved.lock, {
|
|
787
|
+
onPurged: async (_currentTransaction, freedBytes) => {
|
|
788
|
+
purgedAt = new Date().toISOString();
|
|
789
|
+
await mutate(storageRoot, record.id, (current) => {
|
|
790
|
+
if (!matchesReclaimed(current))
|
|
791
|
+
throw new Error("Archive reclaim metadata changed before purge finalization");
|
|
792
|
+
if (current.reclaimed?.purgedAt && hasPlausibleFinalFreedBytes(current.reclaimed, transaction)) {
|
|
793
|
+
purgedAt = current.reclaimed.purgedAt;
|
|
794
|
+
return current;
|
|
795
|
+
}
|
|
796
|
+
const { stagingPath: _stagingPath, ...committed } = current.reclaimed;
|
|
797
|
+
return { ...current, reclaimed: { ...committed, purgedAt, freedBytes } };
|
|
798
|
+
});
|
|
799
|
+
},
|
|
800
|
+
verifyPurgeStillSafe: moved.verifyPurgeStillSafe,
|
|
801
|
+
io: { onYield: operations.cooperate },
|
|
802
|
+
});
|
|
803
|
+
return { freedBytes: finalized.freedBytes, ...(purgedAt ? { purgedAt } : {}) };
|
|
804
|
+
}
|
|
805
|
+
catch (error) {
|
|
806
|
+
const rollbackErrors = [];
|
|
807
|
+
// Once the authorized staging name is gone, the irreversible unlink has
|
|
808
|
+
// happened. Do not infer that from a generic stat failure: only ENOENT is
|
|
809
|
+
// proof of absence. Any other inspection error keeps the journal and all
|
|
810
|
+
// remaining names untouched for reconciliation.
|
|
811
|
+
if (transaction?.stage === "purge-authorized") {
|
|
812
|
+
try {
|
|
813
|
+
await stat(transaction.trashPath);
|
|
814
|
+
}
|
|
815
|
+
catch (stagingError) {
|
|
816
|
+
if (stagingError.code === "ENOENT") {
|
|
817
|
+
throw new AggregateError([error], "Source bytes were purged safely; final metadata will be completed from the durable journal");
|
|
818
|
+
}
|
|
819
|
+
rollbackErrors.push(stagingError);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
// Clear this exact metadata commit before putting the original back. If
|
|
823
|
+
// the clear cannot commit, leave the journaled pointer/staging pair intact;
|
|
824
|
+
// the next run will complete it instead of creating an unauthenticated
|
|
825
|
+
// pointer or overwriting user bytes.
|
|
826
|
+
// A mutation can become durable and then throw (for example its parent
|
|
827
|
+
// directory fsync reports an error after rename). Re-read the exact record
|
|
828
|
+
// after *every* failure; an in-memory boolean cannot distinguish that
|
|
829
|
+
// window from a mutation that never landed.
|
|
830
|
+
let current;
|
|
831
|
+
try {
|
|
832
|
+
current = await readCurrent();
|
|
833
|
+
}
|
|
834
|
+
catch (readError) {
|
|
835
|
+
rollbackErrors.push(readError);
|
|
836
|
+
}
|
|
837
|
+
if (current && isV2(current) && current.reclaimed && !matchesReclaimed(current)) {
|
|
838
|
+
rollbackErrors.push(new Error("Archive reclaim metadata changed before rollback"));
|
|
839
|
+
}
|
|
840
|
+
else if (matchesReclaimed(current)) {
|
|
841
|
+
let clearError;
|
|
842
|
+
await mutate(storageRoot, record.id, (latest) => {
|
|
843
|
+
if (!matchesReclaimed(latest))
|
|
844
|
+
throw new Error("Archive reclaim metadata changed before rollback");
|
|
845
|
+
const { reclaimed: _removed, ...restored } = latest;
|
|
846
|
+
return restored;
|
|
847
|
+
}).catch((error) => { clearError = error; });
|
|
848
|
+
// The clear has the same rename-then-fsync ambiguity as the initial
|
|
849
|
+
// write. Inspect its durable result before deciding whether the byte
|
|
850
|
+
// rollback is safe.
|
|
851
|
+
try {
|
|
852
|
+
const afterClear = await readCurrent();
|
|
853
|
+
if (afterClear && isV2(afterClear) && afterClear.reclaimed) {
|
|
854
|
+
rollbackErrors.push(clearError ?? new Error("Archive reclaim metadata could not be cleared before rollback"));
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
catch (readError) {
|
|
858
|
+
rollbackErrors.push(clearError
|
|
859
|
+
? new AggregateError([clearError, readError], "Could not verify reclaim metadata rollback")
|
|
860
|
+
: readError);
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
if (rollbackErrors.length === 0) {
|
|
864
|
+
if (transaction && moved.lock) {
|
|
865
|
+
await rollbackReclaimTransaction(transaction, moved.lock, { io: { onYield: operations.cooperate } })
|
|
866
|
+
.catch((rollback) => rollbackErrors.push(rollback));
|
|
867
|
+
}
|
|
868
|
+
else {
|
|
869
|
+
await rollbackReclaimMove(record, moved).catch((rollback) => rollbackErrors.push(rollback));
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
if (rollbackErrors.length > 0) {
|
|
873
|
+
throw new AggregateError([error, ...rollbackErrors], "Reclamation failed and could not be rolled back completely");
|
|
874
|
+
}
|
|
875
|
+
throw error;
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
function transactionMatchesReclaim(record, transaction) {
|
|
879
|
+
if (!record || !isV2(record))
|
|
880
|
+
return false;
|
|
881
|
+
const pointer = record.reclaimed?.resumeStub;
|
|
882
|
+
return record.id === transaction.id
|
|
883
|
+
&& record.source.path === transaction.sourcePath
|
|
884
|
+
&& record.source.bytes === transaction.original.bytes
|
|
885
|
+
&& record.source.sha256 === transaction.original.sha256
|
|
886
|
+
&& record.reclaimed?.transactionId === transaction.id
|
|
887
|
+
&& record.reclaimed?.strategy === "purged"
|
|
888
|
+
&& record.reclaimed.originalBytes === transaction.original.bytes
|
|
889
|
+
&& (record.reclaimed.stagingPath === transaction.trashPath
|
|
890
|
+
|| hasPlausibleFinalFreedBytes(record.reclaimed, transaction))
|
|
891
|
+
&& (transaction.replacement ? (pointer?.version === 1
|
|
892
|
+
&& codexSessionIdMatchesFilename(transaction.sourcePath, pointer.sessionId)
|
|
893
|
+
&& pointer.bytes === transaction.replacement.bytes
|
|
894
|
+
&& pointer.sha256 === transaction.replacement.sha256) : pointer === undefined);
|
|
895
|
+
}
|
|
896
|
+
/** Resolve kill -9 windows before starting another native source reclaim. */
|
|
897
|
+
export async function reconcileCodexReclaims(dataDir, storageRoot, home, remoteCheck, cooperate, activity = {}, signal, remoteBytesPerSecond, remoteFetch, authorizeReclaim, phrase, indexOperations = {}) {
|
|
898
|
+
const canonicalHome = home;
|
|
899
|
+
await reconcileReclaimTransactions(dataDir, {
|
|
900
|
+
validateTransaction: async (transaction) => {
|
|
901
|
+
const records = await listArchives(dataDir);
|
|
902
|
+
const record = records.find((candidate) => candidate.id === transaction.id);
|
|
903
|
+
if (!record
|
|
904
|
+
|| record.source.path !== transaction.sourcePath
|
|
905
|
+
|| record.source.bytes !== transaction.original.bytes
|
|
906
|
+
|| record.source.sha256 !== transaction.original.sha256) {
|
|
907
|
+
throw new Error("Reclaim journal does not match its immutable archive source");
|
|
908
|
+
}
|
|
909
|
+
if (transaction.replacement) {
|
|
910
|
+
await validateNativeCodexSessionPath(transaction.sourcePath, canonicalHome, "missing-or-file");
|
|
911
|
+
await validateNativeCodexSessionPath(transaction.replacement.tempPath, canonicalHome, "missing-or-file");
|
|
912
|
+
assertCodexResumePointerTempPath(transaction.replacement.tempPath);
|
|
913
|
+
// A journal is not authority to publish or unlink whichever file its
|
|
914
|
+
// tempPath names. Bind every surviving exact replacement candidate to
|
|
915
|
+
// this archive and logical native session before reconciliation can
|
|
916
|
+
// touch it. Mismatched candidates are retained by generic inode/hash
|
|
917
|
+
// checks; matching-but-malformed candidates fail closed here.
|
|
918
|
+
await validateCodexResumePointerBinding(transaction.replacement.tempPath, transaction.sourcePath, transaction.id, transaction.replacement);
|
|
919
|
+
await validateCodexResumePointerBinding(transaction.sourcePath, transaction.sourcePath, transaction.id, transaction.replacement);
|
|
920
|
+
}
|
|
921
|
+
else {
|
|
922
|
+
const resolvedHome = canonicalHome ?? homedir();
|
|
923
|
+
const roots = record.source.agent === "claude"
|
|
924
|
+
? NATIVE_ROOTS.claude(resolvedHome)
|
|
925
|
+
: record.source.agent === "codex"
|
|
926
|
+
? NATIVE_ROOTS.codex(resolvedHome)
|
|
927
|
+
: [];
|
|
928
|
+
if (roots.length === 0)
|
|
929
|
+
throw new Error(`No safe native reclaim root for agent ${record.source.agent}`);
|
|
930
|
+
await validateNoFollowPath(transaction.sourcePath, roots, { leaf: "missing-or-file" });
|
|
931
|
+
}
|
|
932
|
+
},
|
|
933
|
+
validatePurgeCandidate: async (transaction) => {
|
|
934
|
+
const staged = await stat(transaction.trashPath).catch((error) => {
|
|
935
|
+
if (error.code === "ENOENT")
|
|
936
|
+
return null;
|
|
937
|
+
throw error;
|
|
938
|
+
});
|
|
939
|
+
// A daemon restart is not authority to finish an earlier deletion after
|
|
940
|
+
// the person has since turned reclaim off or selected a safe policy. The
|
|
941
|
+
// sole exception is a purge-authorized journal whose staged inode is
|
|
942
|
+
// already absent: the irreversible unlink happened before the crash, so
|
|
943
|
+
// reconciliation is only finishing truthful metadata and journal cleanup.
|
|
944
|
+
if (staged || transaction.stage !== "purge-authorized") {
|
|
945
|
+
await assertReclaimAuthorized(authorizeReclaim, transaction.id);
|
|
946
|
+
}
|
|
947
|
+
if (!transaction.replacement)
|
|
948
|
+
return;
|
|
949
|
+
// An old binary may have staged a multi-agent child before this release
|
|
950
|
+
// learned that installed Codex cannot resume it independently. Re-parse
|
|
951
|
+
// whichever exact original name still survives, against the logical
|
|
952
|
+
// rollout filename, before allowing reconciliation to advance. If an
|
|
953
|
+
// authorized purge already completed there are no bytes left to restore;
|
|
954
|
+
// final metadata can only be completed conservatively.
|
|
955
|
+
const contentsPath = staged?.isFile()
|
|
956
|
+
? transaction.trashPath
|
|
957
|
+
: transaction.stage === "prepared"
|
|
958
|
+
? transaction.sourcePath
|
|
959
|
+
: null;
|
|
960
|
+
if (contentsPath) {
|
|
961
|
+
await assertCodexResumePointerMetadataSupported(contentsPath, transaction.sourcePath);
|
|
962
|
+
}
|
|
963
|
+
},
|
|
964
|
+
isMetadataCommitted: async (transaction) => {
|
|
965
|
+
const records = await listArchives(dataDir);
|
|
966
|
+
return transactionMatchesReclaim(records.find((record) => record.id === transaction.id), transaction);
|
|
967
|
+
},
|
|
968
|
+
clearMetadataCommit: async (transaction) => {
|
|
969
|
+
await mutateArchiveRecord(storageRoot, transaction.id, (current) => {
|
|
970
|
+
if (!transactionMatchesReclaim(current, transaction))
|
|
971
|
+
throw new Error("Archive reclaim metadata changed before crash rollback");
|
|
972
|
+
const { reclaimed: _removed, ...restored } = current;
|
|
973
|
+
return restored;
|
|
974
|
+
});
|
|
975
|
+
},
|
|
976
|
+
verifyPurgeStillSafe: async (transaction) => {
|
|
977
|
+
// A crash may be reconciled minutes or days later, including after the
|
|
978
|
+
// purge-authorization record landed. Authorization is not a lease on an
|
|
979
|
+
// external object, so repeat the provider proof whenever staged bytes
|
|
980
|
+
// still exist immediately before their unlink.
|
|
981
|
+
const publication = await verifyCurrentDurabilityImmediatelyBeforePurge(dataDir, transaction.id, transaction.sourcePath, remoteCheck, activity, {
|
|
982
|
+
signal,
|
|
983
|
+
cooperate,
|
|
984
|
+
bytesPerSecond: remoteBytesPerSecond,
|
|
985
|
+
fetchRemoteCopy: remoteFetch,
|
|
986
|
+
phrase,
|
|
987
|
+
verifyIndexPublication: indexOperations.verifyIndexPublication,
|
|
988
|
+
});
|
|
989
|
+
const finalStagedActivity = await isSourceActive(transaction.trashPath, {
|
|
990
|
+
...activity,
|
|
991
|
+
now: Date.now(),
|
|
992
|
+
});
|
|
993
|
+
if (finalStagedActivity.active) {
|
|
994
|
+
throw new Error(`The staged session is still open: ${finalStagedActivity.reason}`);
|
|
995
|
+
}
|
|
996
|
+
await assertReclaimAuthorized(authorizeReclaim, transaction.id);
|
|
997
|
+
await (indexOperations.recheckIndexPublication
|
|
998
|
+
? indexOperations.recheckIndexPublication(dataDir, publication, signal)
|
|
999
|
+
: recheckManagedAccountIndexPublicationBeforeUnlink({ dataDir, verification: publication, signal }));
|
|
1000
|
+
},
|
|
1001
|
+
ensureAuditCommitted: async (transaction) => {
|
|
1002
|
+
await recordAudit(dataDir, "source.reclaim", "allowed", {
|
|
1003
|
+
archiveId: transaction.id,
|
|
1004
|
+
path: transaction.sourcePath,
|
|
1005
|
+
transactionId: transaction.id,
|
|
1006
|
+
strategy: "purged",
|
|
1007
|
+
purgeAuthorized: true,
|
|
1008
|
+
expectedFreedBytes: transactionAllocatedBytesFreed(transaction),
|
|
1009
|
+
resumeStub: Boolean(transaction.replacement),
|
|
1010
|
+
reconciled: true,
|
|
1011
|
+
});
|
|
1012
|
+
},
|
|
1013
|
+
onPurged: async (transaction, freedBytes) => {
|
|
1014
|
+
await mutateArchiveRecord(storageRoot, transaction.id, (current) => {
|
|
1015
|
+
if (!transactionMatchesReclaim(current, transaction)) {
|
|
1016
|
+
throw new Error("Archive reclaim metadata changed before crash finalization");
|
|
1017
|
+
}
|
|
1018
|
+
if (current.reclaimed?.purgedAt && hasPlausibleFinalFreedBytes(current.reclaimed, transaction))
|
|
1019
|
+
return current;
|
|
1020
|
+
const { stagingPath: _stagingPath, ...committed } = current.reclaimed;
|
|
1021
|
+
return {
|
|
1022
|
+
...current,
|
|
1023
|
+
reclaimed: { ...committed, purgedAt: new Date().toISOString(), freedBytes },
|
|
1024
|
+
};
|
|
1025
|
+
});
|
|
1026
|
+
},
|
|
1027
|
+
io: { onYield: cooperate },
|
|
1028
|
+
});
|
|
1029
|
+
}
|
|
178
1030
|
/**
|
|
179
1031
|
* Reclaims eligible sources. Dry-run is the default: without `confirm`, nothing on
|
|
180
|
-
* disk changes.
|
|
1032
|
+
* disk changes. Confirmed, verified sources are transaction-staged and then
|
|
1033
|
+
* permanently purged; Codex receives a tiny same-id resume pointer.
|
|
181
1034
|
*/
|
|
182
1035
|
export async function applyRetention(dataDir, options = {}) {
|
|
183
1036
|
const config = await readConfig(dataDir);
|
|
1037
|
+
const cooperate = options.operations?.cooperate ?? (options.backgroundCpuTarget === undefined
|
|
1038
|
+
? undefined
|
|
1039
|
+
: createBackgroundCpuYield({ target: options.backgroundCpuTarget, signal: options.signal }));
|
|
1040
|
+
// Reconciliation runs before candidate evaluation because it must repair a
|
|
1041
|
+
// previous kill -9 window first. That ordering must not turn a preview into
|
|
1042
|
+
// deletion authority. A confirmed foreground pass is bounded by the current
|
|
1043
|
+
// persisted policy (and current per-id approval); a daemon pass must satisfy
|
|
1044
|
+
// that policy AND its machine-local live gate.
|
|
1045
|
+
const authorizeReclaim = options.confirm === true
|
|
1046
|
+
? confirmedRetentionAuthorization(dataDir, options.authorizeReclaim)
|
|
1047
|
+
: denyReclaimAuthorization;
|
|
1048
|
+
await reconcileCodexReclaims(dataDir, config.storage.root, options.home, options.operations?.checkRemoteCopy, cooperate, options.activity, options.signal, options.backgroundRemoteBytesPerSecond, options.operations?.fetchRemoteCopy, authorizeReclaim, options.phrase, options.operations);
|
|
184
1049
|
const evaluation = await evaluateRetention(dataDir, options);
|
|
185
1050
|
const eligible = evaluation.candidates.filter((candidate) => candidate.eligible);
|
|
186
1051
|
if (!options.confirm) {
|
|
187
|
-
return {
|
|
1052
|
+
return {
|
|
1053
|
+
mode: "dry-run", policy: evaluation.policy, wouldReclaim: eligible.length,
|
|
1054
|
+
wouldFreeBytes: evaluation.reclaimableBytes,
|
|
1055
|
+
wouldMoveBytes: eligible.reduce((total, candidate) => total + candidate.bytes, 0),
|
|
1056
|
+
candidates: evaluation.candidates
|
|
1057
|
+
};
|
|
188
1058
|
}
|
|
189
1059
|
const records = new Map((await listArchives(dataDir)).map((record) => [record.id, record]));
|
|
190
1060
|
const reclaimed = [];
|
|
191
1061
|
const failed = [];
|
|
192
1062
|
for (const candidate of eligible) {
|
|
1063
|
+
// Stop the batch on a revoked/unreadable live authorization. Continuing to
|
|
1064
|
+
// inspect hundreds of candidates cannot become safer within this pass.
|
|
193
1065
|
try {
|
|
194
|
-
|
|
1066
|
+
await assertReclaimAuthorized(authorizeReclaim, candidate.id);
|
|
1067
|
+
}
|
|
1068
|
+
catch (error) {
|
|
1069
|
+
failed.push({ id: candidate.id, error: error instanceof Error ? error.message : "automatic reclaim is no longer allowed" });
|
|
1070
|
+
break;
|
|
1071
|
+
}
|
|
1072
|
+
try {
|
|
1073
|
+
// The evaluation ran one lsof per candidate before this loop — a wide
|
|
1074
|
+
// window on a long batch. What is moved must still be what was verified.
|
|
195
1075
|
const record = records.get(candidate.id);
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
1076
|
+
const remoteChain = completeRemoteChain([...records.values()], record);
|
|
1077
|
+
if (!remoteChain.complete)
|
|
1078
|
+
throw new Error("The complete remote archive chain changed before reclaim; the local original was kept");
|
|
1079
|
+
const lock = await acquireReclaimSourceLock(dataDir, record.source.path);
|
|
1080
|
+
try {
|
|
1081
|
+
await verifySourceImmediatelyBeforeReclaim(record, cooperate);
|
|
1082
|
+
const moved = await moveReclaimableSource(dataDir, record, {
|
|
1083
|
+
home: options.home,
|
|
1084
|
+
strategy: options.strategy,
|
|
1085
|
+
operations: options.operations,
|
|
1086
|
+
lock,
|
|
1087
|
+
activity: options.activity,
|
|
1088
|
+
now: options.now,
|
|
1089
|
+
remoteChain: remoteChain.chain,
|
|
1090
|
+
cooperate,
|
|
1091
|
+
signal: options.signal,
|
|
1092
|
+
remoteBytesPerSecond: options.backgroundRemoteBytesPerSecond,
|
|
1093
|
+
phrase: options.phrase,
|
|
1094
|
+
authorizeReclaim,
|
|
1095
|
+
assertStillAllowed: async () => {
|
|
1096
|
+
if (isPinned(record.source.path, await neverReclaimMarks(dataDir))) {
|
|
1097
|
+
fail("invalid_argument", "This session was pinned while reclaim was running, so the original was kept.", { archiveId: record.id });
|
|
1098
|
+
}
|
|
1099
|
+
const stillActive = await isSourceActive(record.source.path, {
|
|
1100
|
+
...options.activity,
|
|
1101
|
+
now: options.now ?? Date.now(),
|
|
1102
|
+
});
|
|
1103
|
+
if (stillActive.active) {
|
|
1104
|
+
fail("invalid_argument", `The session became active while reclaim was running: ${stillActive.reason}. The original was kept.`, { archiveId: record.id });
|
|
1105
|
+
}
|
|
1106
|
+
},
|
|
1107
|
+
});
|
|
1108
|
+
const result = moved.trash;
|
|
1109
|
+
if (isV2(record)) {
|
|
1110
|
+
const reclaimedAt = new Date(options.now ?? Date.now()).toISOString();
|
|
1111
|
+
const committed = await commitReclaim(dataDir, config.storage.root, record, moved, reclaimedAt, {
|
|
1112
|
+
archiveId: candidate.id,
|
|
1113
|
+
path: candidate.path,
|
|
1114
|
+
bytes: candidate.bytes,
|
|
1115
|
+
stagingPath: result.movedTo,
|
|
1116
|
+
strategy: moved.transaction ? "purged" : result.strategy,
|
|
1117
|
+
...(moved.stub ? { resumeStub: true, sessionId: moved.stub.sessionId, resumeStubBytes: moved.stub.bytes } : {}),
|
|
1118
|
+
}, options.operations);
|
|
1119
|
+
reclaimed.push({
|
|
1120
|
+
id: candidate.id,
|
|
1121
|
+
path: candidate.path,
|
|
1122
|
+
bytes: candidate.bytes,
|
|
1123
|
+
movedTo: committed.purgedAt ? "" : result.movedTo,
|
|
1124
|
+
strategy: committed.purgedAt ? "purged" : result.strategy,
|
|
1125
|
+
freedBytes: committed.freedBytes,
|
|
1126
|
+
...(committed.purgedAt ? { purgedAt: committed.purgedAt } : {}),
|
|
1127
|
+
...(moved.stub ? { resumeStub: { sessionId: moved.stub.sessionId, bytes: moved.stub.bytes } } : {}),
|
|
1128
|
+
});
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
finally {
|
|
1132
|
+
await lock.release();
|
|
199
1133
|
}
|
|
200
|
-
await recordAudit(dataDir, "source.reclaim", "allowed", { archiveId: candidate.id, path: candidate.path, bytes: candidate.bytes, movedTo: result.movedTo, strategy: result.strategy });
|
|
201
|
-
reclaimed.push({ id: candidate.id, path: candidate.path, bytes: candidate.bytes, movedTo: result.movedTo, strategy: result.strategy });
|
|
202
1134
|
}
|
|
203
1135
|
catch (error) {
|
|
204
|
-
await recordAudit(dataDir, "source.reclaim", "denied", { archiveId: candidate.id, path: candidate.path });
|
|
1136
|
+
await (options.operations?.audit ?? recordAudit)(dataDir, "source.reclaim", "denied", { archiveId: candidate.id, path: candidate.path }).catch(() => undefined);
|
|
205
1137
|
failed.push({ id: candidate.id, error: error instanceof Error ? error.message : "reclamation failed" });
|
|
206
1138
|
}
|
|
207
1139
|
}
|
|
208
|
-
|
|
1140
|
+
const movedBytes = reclaimed.reduce((total, item) => total + item.bytes, 0);
|
|
1141
|
+
const freedBytes = reclaimed.reduce((total, item) => total + item.freedBytes, 0);
|
|
1142
|
+
return {
|
|
1143
|
+
mode: "apply", policy: evaluation.policy, reclaimed, failed,
|
|
1144
|
+
freedBytes, movedBytes, protected: evaluation.candidates.length - eligible.length
|
|
1145
|
+
};
|
|
209
1146
|
}
|
|
210
1147
|
/**
|
|
211
1148
|
* Reclaims ONE source, because a person pointed at it.
|
|
@@ -218,16 +1155,25 @@ export async function applyRetention(dataDir, options = {}) {
|
|
|
218
1155
|
*
|
|
219
1156
|
* - the archive must have a verified remote copy (and a delta head's whole
|
|
220
1157
|
* chain must be verified — one unverified base and the cloud cannot
|
|
221
|
-
* reproduce the file this is about to
|
|
1158
|
+
* reproduce the file this is about to reclaim);
|
|
1159
|
+
* - this exact archive must be covered by the search index, so durable memory
|
|
1160
|
+
* is also memory a fresh agent can actually find;
|
|
222
1161
|
* - the source must be byte-for-byte the file that was archived — a session
|
|
223
1162
|
* that grew since has content in no archive at all;
|
|
224
1163
|
* - nothing may have the file open right now.
|
|
225
1164
|
*
|
|
226
|
-
* Policy, age and grace are deliberately not consulted.
|
|
227
|
-
*
|
|
1165
|
+
* Policy, age and grace are deliberately not consulted. The explicit click is
|
|
1166
|
+
* the irreversible-delete approval; all durability/search/activity gates still
|
|
1167
|
+
* run immediately before the transaction commits.
|
|
228
1168
|
*/
|
|
229
1169
|
export async function reclaimOne(dataDir, archiveId, options = {}) {
|
|
230
1170
|
const config = await readConfig(dataDir);
|
|
1171
|
+
const cooperate = options.operations?.cooperate;
|
|
1172
|
+
// A direct confirmed click is authority for exactly the selected archive,
|
|
1173
|
+
// never for another crash journal found beside it. An unconfirmed request is
|
|
1174
|
+
// a preview/refusal and may only drive conservative rollback.
|
|
1175
|
+
const authorizeReconciliation = async (pendingArchiveId) => options.confirm === true && pendingArchiveId === archiveId;
|
|
1176
|
+
await reconcileCodexReclaims(dataDir, config.storage.root, options.home, options.operations?.checkRemoteCopy, cooperate, options.activity, undefined, undefined, options.operations?.fetchRemoteCopy, authorizeReconciliation, options.phrase, options.operations);
|
|
231
1177
|
const archives = await listArchives(dataDir);
|
|
232
1178
|
const record = archives.find((item) => item.id === archiveId);
|
|
233
1179
|
if (!record)
|
|
@@ -235,40 +1181,94 @@ export async function reclaimOne(dataDir, archiveId, options = {}) {
|
|
|
235
1181
|
if (!isV2(record))
|
|
236
1182
|
fail("invalid_argument", `Archive ${archiveId} uses format v1. Run: sealkeep migrate`, { archiveId });
|
|
237
1183
|
if (record.reclaimed)
|
|
238
|
-
fail("invalid_argument", "This source was already reclaimed
|
|
1184
|
+
fail("invalid_argument", "This source was already reclaimed from this machine.");
|
|
239
1185
|
const current = await stat(record.source.path).catch(() => null);
|
|
240
1186
|
if (!current)
|
|
241
1187
|
fail("source_unreadable", "The source file is no longer at its recorded path, so there is nothing to reclaim.", { path: record.source.path });
|
|
242
1188
|
if (current.size !== record.source.bytes) {
|
|
243
1189
|
fail("invalid_argument", `The session changed since it was archived (${record.source.bytes} bytes archived, ${current.size} now) — re-archive it first, or reclaiming would delete exactly the part no archive holds.`, { archiveId });
|
|
244
1190
|
}
|
|
245
|
-
|
|
246
|
-
|
|
1191
|
+
const remoteChain = completeRemoteChain(archives, record);
|
|
1192
|
+
if (!remoteChain.complete) {
|
|
1193
|
+
const unverified = remoteChain.chain.filter((link) => !(isV2(link) && hasVerifiedCopy(link))).length;
|
|
1194
|
+
fail("invalid_argument", remoteChain.chain.length === 0
|
|
1195
|
+
? "This snapshot builds on an earlier archive that is missing, so the cloud cannot reproduce the file. Keep the source."
|
|
1196
|
+
: unverified > 0
|
|
1197
|
+
? "No verified cloud copy exists for every archive in this snapshot's chain, so this disk may hold the only bytes. Upload the complete chain first."
|
|
1198
|
+
: "This snapshot's verified copies are split across storage destinations, so no one destination can reproduce the file. Upload the complete chain to one destination first.", { archiveId });
|
|
247
1199
|
}
|
|
248
|
-
const
|
|
249
|
-
if (
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
if (unverified > 0)
|
|
254
|
-
fail("invalid_argument", `This snapshot builds on earlier archives and ${unverified} of the chain ${unverified === 1 ? "is" : "are"} not verified in the cloud yet — the cloud cannot reproduce this file without them.`, { archiveId });
|
|
255
|
-
}
|
|
256
|
-
catch (error) {
|
|
257
|
-
if (isVaultlineError(error))
|
|
258
|
-
throw error;
|
|
259
|
-
fail("invalid_argument", "This snapshot builds on an earlier archive that is missing, so the cloud cannot reproduce the file. Keep the source.", { archiveId });
|
|
260
|
-
}
|
|
1200
|
+
const indexed = await indexedArchiveIds(dataDir, archives);
|
|
1201
|
+
if (!indexed?.has(record.id)) {
|
|
1202
|
+
fail("invalid_argument", indexed
|
|
1203
|
+
? "This archive is safely stored but not searchable yet. Sealkeep is still indexing it, so the original stays on this disk for now."
|
|
1204
|
+
: "Sealkeep could not confirm this archive is in the search index, so the original stays on this disk.", { archiveId });
|
|
261
1205
|
}
|
|
262
1206
|
const activity = await isSourceActive(record.source.path, { ...options.activity, now: options.now ?? Date.now() });
|
|
263
1207
|
if (activity.active)
|
|
264
1208
|
fail("invalid_argument", `The file looks in use right now: ${activity.reason}. Close it and try again.`, { archiveId });
|
|
265
1209
|
if (!options.confirm)
|
|
266
|
-
fail("invalid_argument", "Pass confirm to
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
1210
|
+
fail("invalid_argument", "Pass confirm to permanently free this source's local disk space.");
|
|
1211
|
+
if (isPinned(record.source.path, await neverReclaimMarks(dataDir))) {
|
|
1212
|
+
fail("invalid_argument", "This session is listed in never-reclaim — pinned on purpose. Edit <dataDir>/never-reclaim to release it.", { archiveId });
|
|
1213
|
+
}
|
|
1214
|
+
/**
|
|
1215
|
+
* Re-checked at the last instant. The size gate above ran BEFORE the
|
|
1216
|
+
* activity probe (which shells out to lsof — real wall-clock); an agent
|
|
1217
|
+
* appending in that window could otherwise have its never-archived tail swept
|
|
1218
|
+
* away with the verified part. What is purged must be what was verified.
|
|
1219
|
+
*/
|
|
1220
|
+
const lock = await acquireReclaimSourceLock(dataDir, record.source.path);
|
|
1221
|
+
try {
|
|
1222
|
+
await verifySourceImmediatelyBeforeReclaim(record, cooperate);
|
|
1223
|
+
const moved = await moveReclaimableSource(dataDir, record, {
|
|
1224
|
+
home: options.home,
|
|
1225
|
+
strategy: options.strategy,
|
|
1226
|
+
operations: options.operations,
|
|
1227
|
+
lock,
|
|
1228
|
+
activity: options.activity,
|
|
1229
|
+
now: options.now,
|
|
1230
|
+
remoteChain: remoteChain.chain,
|
|
1231
|
+
cooperate,
|
|
1232
|
+
signal: options.signal,
|
|
1233
|
+
phrase: options.phrase,
|
|
1234
|
+
assertStillAllowed: async () => {
|
|
1235
|
+
if (isPinned(record.source.path, await neverReclaimMarks(dataDir))) {
|
|
1236
|
+
fail("invalid_argument", "This session was pinned while reclaim was running, so the original was kept.", { archiveId });
|
|
1237
|
+
}
|
|
1238
|
+
const stillActive = await isSourceActive(record.source.path, {
|
|
1239
|
+
...options.activity,
|
|
1240
|
+
now: options.now ?? Date.now(),
|
|
1241
|
+
});
|
|
1242
|
+
if (stillActive.active) {
|
|
1243
|
+
fail("invalid_argument", `The session became active while reclaim was running: ${stillActive.reason}. The original was kept.`, { archiveId });
|
|
1244
|
+
}
|
|
1245
|
+
},
|
|
1246
|
+
});
|
|
1247
|
+
const result = moved.trash;
|
|
1248
|
+
const reclaimedAt = new Date(options.now ?? Date.now()).toISOString();
|
|
1249
|
+
const committed = await commitReclaim(dataDir, config.storage.root, record, moved, reclaimedAt, {
|
|
1250
|
+
archiveId,
|
|
1251
|
+
path: record.source.path,
|
|
1252
|
+
bytes: record.source.bytes,
|
|
1253
|
+
stagingPath: result.movedTo,
|
|
1254
|
+
strategy: moved.transaction ? "purged" : result.strategy,
|
|
1255
|
+
manual: true,
|
|
1256
|
+
...(moved.stub ? { resumeStub: true, sessionId: moved.stub.sessionId, resumeStubBytes: moved.stub.bytes } : {}),
|
|
1257
|
+
}, options.operations);
|
|
1258
|
+
return {
|
|
1259
|
+
id: archiveId,
|
|
1260
|
+
path: record.source.path,
|
|
1261
|
+
bytes: record.source.bytes,
|
|
1262
|
+
movedTo: committed.purgedAt ? "" : result.movedTo,
|
|
1263
|
+
strategy: committed.purgedAt ? "purged" : result.strategy,
|
|
1264
|
+
freedBytes: committed.freedBytes,
|
|
1265
|
+
...(committed.purgedAt ? { purgedAt: committed.purgedAt } : {}),
|
|
1266
|
+
...(moved.stub ? { resumeStub: { sessionId: moved.stub.sessionId, bytes: moved.stub.bytes } } : {}),
|
|
1267
|
+
};
|
|
1268
|
+
}
|
|
1269
|
+
finally {
|
|
1270
|
+
await lock.release();
|
|
1271
|
+
}
|
|
272
1272
|
}
|
|
273
1273
|
/**
|
|
274
1274
|
* `reclaimOne` over a selection. Each row passes or is kept with its reason —
|
|
@@ -286,7 +1286,13 @@ export async function reclaimBatch(dataDir, ids, options = {}) {
|
|
|
286
1286
|
kept.push({ id, reason: error instanceof Error ? error.message : "could not reclaim" });
|
|
287
1287
|
}
|
|
288
1288
|
}
|
|
289
|
-
|
|
1289
|
+
const movedBytes = reclaimed.reduce((total, item) => total + item.bytes, 0);
|
|
1290
|
+
return {
|
|
1291
|
+
reclaimed,
|
|
1292
|
+
kept,
|
|
1293
|
+
freedBytes: reclaimed.reduce((total, item) => total + item.freedBytes, 0),
|
|
1294
|
+
movedBytes,
|
|
1295
|
+
};
|
|
290
1296
|
}
|
|
291
1297
|
export async function setRetentionPolicy(dataDir, settings) {
|
|
292
1298
|
if (settings.policy && !POLICIES.includes(settings.policy))
|
|
@@ -304,9 +1310,7 @@ export async function setRetentionPolicy(dataDir, settings) {
|
|
|
304
1310
|
fail("invalid_argument", `${key} must be a non-negative number, or null to ignore it`);
|
|
305
1311
|
}
|
|
306
1312
|
const config = await readConfig(dataDir);
|
|
307
|
-
|
|
308
|
-
await writeConfig(dataDir, next);
|
|
309
|
-
return next;
|
|
1313
|
+
return updateConfigFields(dataDir, { retention: retentionSettings(config, settings) });
|
|
310
1314
|
}
|
|
311
1315
|
/**
|
|
312
1316
|
* Snapshots whose every byte is already inside a newer archive of the same file.
|
|
@@ -369,7 +1373,7 @@ export async function redundantArchives(dataDir) {
|
|
|
369
1373
|
const heirObject = await stat(heir.objectPath).catch(() => null);
|
|
370
1374
|
if (!heirObject)
|
|
371
1375
|
continue;
|
|
372
|
-
if (config.remoteStorage && !(isV2(heir) && heir
|
|
1376
|
+
if (config.remoteStorage && !(isV2(heir) && hasVerifiedCopy(heir)))
|
|
373
1377
|
continue;
|
|
374
1378
|
const own = await stat(record.objectPath).catch(() => null);
|
|
375
1379
|
if (!own)
|