sliftutils 1.7.102 → 1.7.104

package/storage/remoteStorage/storageController.d.ts

@@ -3,6 +3,7 @@
 import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, FindConfig, SourceConfig } from "../IArchives";
 import { ActiveBucketInfo, ServerBucketInfo } from "./storageServerState";
 import { AccessTotals, AccessSummaryState } from "./accessStats";
+import { LogFileInfo } from "../StreamingLogs";
 import type { SummaryEntry } from "../../treeSummary";
 export declare const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
 export declare const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
@@ -84,6 +85,7 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
         };
         internal?: boolean;
         includeTombstones?: boolean;
+        includeMarked?: boolean;
     }) => Promise<{
         data: Buffer;
         writeTime: number;
@@ -98,6 +100,7 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
         lastModified?: number;
         forceSetImmutable?: boolean;
         internal?: boolean;
+        undelete?: boolean;
     }) => Promise<void>;
     del: (config: {
         account: string;
@@ -182,6 +185,13 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
         account: string;
         bucketName: string;
     }) => Promise<ArchivesSyncStatus>;
+    listLogFiles: (config: {
+        account: string;
+    }) => Promise<LogFileInfo[]>;
+    getLogFile: (config: {
+        account: string;
+        name: string;
+    }) => Promise<Buffer>;
     startLargeFile: (config: {
         account: string;
         bucketName: string;

package/storage/remoteStorage/storageController.ts

@@ -20,6 +20,8 @@ import { BlobStore } from "./blobStore";
 import { getRoutingFileResult } from "./bucketDisk";
 import { StorageClientController } from "./storageClientController";
 import { trackAccess, trackAccessCall, getAccessTotals, readAccessSummaries, clearAccountAccessStats, AccessTotals, AccessSummaryState } from "./accessStats";
+import { logMutation, listStorageLogFiles, readStorageLogFile } from "./storageLogs";
+import { LogFileInfo } from "../StreamingLogs";
 import { assertValidName, assertValidPath, assertValidArgs } from "./validation";
 import type { SummaryEntry } from "../../treeSummary";
 
@@ -275,7 +277,7 @@ class RemoteStorageControllerBase {
 
     @assertValidArgs
     @trackAccessCall("get")
-    async get2(config: { account: string; bucketName: string; path: string; sourceConfig: SourceConfig; range?: { start: number; end: number }; internal?: boolean; includeTombstones?: boolean }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
+    async get2(config: { account: string; bucketName: string; path: string; sourceConfig: SourceConfig; range?: { start: number; end: number }; internal?: boolean; includeTombstones?: boolean; includeMarked?: boolean }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
         if (config.path === ROUTING_FILE) {
             // The routing file lives outside every store and even outside the bucket (it is what CREATES it), so it is read straight off the disk - absent means undefined, exactly like any file read
             return await getRoutingFileResult(config.account, config.bucketName);
@@ -287,8 +289,9 @@ class RemoteStorageControllerBase {
     }
     @assertValidArgs
     @trackAccessCall("set")
-    async set(config: { account: string; bucketName: string; path: string; data: Buffer; sourceConfig: SourceConfig; lastModified?: number; forceSetImmutable?: boolean; internal?: boolean }): Promise<void> {
+    async set(config: { account: string; bucketName: string; path: string; data: Buffer; sourceConfig: SourceConfig; lastModified?: number; forceSetImmutable?: boolean; internal?: boolean; undelete?: boolean }): Promise<void> {
         assertWritesAllowed();
+        let caller = callerId();
         // Copied because the wire hands us a plain Uint8Array view, not a real Buffer
         let data = Buffer.from(config.data);
         if (config.path === ROUTING_FILE) {
@@ -296,21 +299,26 @@ class RemoteStorageControllerBase {
             return await writeRoutingConfig(config.account, config.bucketName, config.sourceConfig.name, data, config);
         }
         await withStore(config, store => store.set({ ...config, data }));
+        logMutation({ op: config.undelete && "undelete" || "set", account: config.account, bucketName: config.bucketName, store: config.sourceConfig.name, path: config.path, size: data.length, writeTime: config.lastModified, callerId: caller, internal: config.internal });
     }
     @assertValidArgs
     @trackAccessCall("del")
     async del(config: { account: string; bucketName: string; path: string; sourceConfig: SourceConfig; lastModified?: number; internal?: boolean }): Promise<void> {
         assertWritesAllowed();
+        let caller = callerId();
         await withStore(config, store => store.del(config));
+        logMutation({ op: "del", account: config.account, bucketName: config.bucketName, store: config.sourceConfig.name, path: config.path, writeTime: config.lastModified, callerId: caller, internal: config.internal });
     }
     @assertValidArgs
     @trackAccessCall("move")
     async move(config: { account: string; bucketName: string; fromPath: string; toPath: string; sourceConfig: SourceConfig }): Promise<void> {
         assertWritesAllowed();
+        let caller = callerId();
         // assertValidArgs only knows the well-known `path` field, so the two paths of a move are validated here
         assertValidPath(config.fromPath);
         assertValidPath(config.toPath);
         await withStore(config, store => store.move(config));
+        logMutation({ op: "move", account: config.account, bucketName: config.bucketName, store: config.sourceConfig.name, path: config.fromPath, toPath: config.toPath, callerId: caller });
     }
     @assertValidArgs
     @trackAccessCall("getInfo")
@@ -380,6 +388,17 @@ class RemoteStorageControllerBase {
         return await bucketSyncStatus(config.account, config.bucketName);
     }
 
+    /** The operation-log files THIS server holds (every server logs its own operations - ask each one; see listAllServerLogFiles in createArchives). */
+    @assertValidArgs
+    async listLogFiles(config: { account: string }): Promise<LogFileInfo[]> {
+        return await listStorageLogFiles();
+    }
+    /** One log file's bytes, always LZ4-compressed (a live file is flushed and compressed in memory before sending). Decode with decodeLogFile. */
+    @assertValidArgs
+    async getLogFile(config: { account: string; name: string }): Promise<Buffer> {
+        return await readStorageLogFile(config.name);
+    }
+
     @assertValidArgs
     async startLargeFile(config: { account: string; bucketName: string; path: string; sourceConfig: SourceConfig; lastModified?: number; forceSetImmutable?: boolean; noChecks?: boolean; internal?: boolean }): Promise<string> {
         assertWritesAllowed();
@@ -404,7 +423,11 @@ class RemoteStorageControllerBase {
         let info = largeUploadInfo.get(config.uploadId);
         if (!info) throw new Error(`Unknown large upload ${config.uploadId}`);
         largeUploadInfo.delete(config.uploadId);
-        await getStore(info.account, info.bucketName, info.storeName).finishLargeUpload({ id: config.uploadId, path: info.path, lastModified: info.lastModified, forceSetImmutable: info.forceSetImmutable, noChecks: info.noChecks, internal: info.internal });
+        let caller = callerId();
+        let store = getStore(info.account, info.bucketName, info.storeName);
+        await store.finishLargeUpload({ id: config.uploadId, path: info.path, lastModified: info.lastModified, forceSetImmutable: info.forceSetImmutable, noChecks: info.noChecks, internal: info.internal });
+        let written = await store.getInfo({ path: info.path });
+        logMutation({ op: "setLarge", account: info.account, bucketName: info.bucketName, store: info.storeName, path: info.path, size: written?.size, writeTime: written?.writeTime, callerId: caller });
     }
     /** Best-effort cleanup: an unknown upload has nothing left to cancel. */
     async cancelLargeFile(config: { uploadId: string }): Promise<void> {
@@ -527,6 +550,15 @@ async function withStore<T>(config: { account: string; bucketName: string; sourc
     return await fn(findBucketStore(config.account, config.bucketName, config.sourceConfig));
 }
 
+// The caller's node id for the mutation log - read synchronously at method entry (the call context does not survive awaits), and tolerant of there being none (local calls)
+function callerId(): string | undefined {
+    try {
+        return SocketFunction.getCaller()?.nodeId;
+    } catch {
+        return undefined;
+    }
+}
+
 const largeUploadInfo = new Map<string, { account: string; bucketName: string; path: string; lastModified?: number; forceSetImmutable?: boolean; noChecks?: boolean; internal?: boolean; storeName: string }>();
 
 const accountAccess: SocketFunctionHook = async (context) => {
@@ -577,6 +609,8 @@ export const RemoteStorageController = SocketFunction.register(
         getAccessStats: { hooks: [accountAccess] },
         getAccessSummaries: { hooks: [accountAccess] },
         getSyncStatus: { hooks: [accountAccess] },
+        listLogFiles: { hooks: [accountAccess] },
+        getLogFile: { hooks: [accountAccess] },
         startLargeFile: { hooks: [accountAccess] },
         uploadPart: { hooks: [uploadAccess] },
         finishLargeFile: { hooks: [uploadAccess] },

package/storage/remoteStorage/storageLogs.d.ts

@@ -0,0 +1,29 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { LogFileInfo } from "../StreamingLogs";
+export declare const LOGS_FOLDER_NAME = "logs";
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. */
+export declare function logMutation(entry: {
+    op: string;
+    account: string;
+    bucketName: string;
+    store?: string;
+    path: string;
+    toPath?: string;
+    size?: number;
+    writeTime?: number;
+    callerId?: string;
+    internal?: boolean;
+}): void;
+/** A synchronization key point: scans and full syncs starting/finishing, reconciles, boundary scans - what an operator greps for to see whether the fleet is converging. */
+export declare function logSyncEvent(entry: {
+    event: string;
+    store: string;
+    source?: string;
+    [key: string]: unknown;
+}): void;
+export declare function logStorageError(message: string): void;
+/** The log files this server holds - see StreamingLogs.listFiles. Empty on processes with no storage folder. */
+export declare function listStorageLogFiles(): Promise<LogFileInfo[]>;
+/** One log file's bytes, always LZ4-compressed - see StreamingLogs.readFileCompressed (decode with decodeLogFile). */
+export declare function readStorageLogFile(name: string): Promise<Buffer>;

package/storage/remoteStorage/storageLogs.ts

@@ -0,0 +1,96 @@
+import os from "os";
+import path from "path";
+import { lazy } from "socket-function/src/caching";
+import { StreamingLogs, LogFileInfo } from "../StreamingLogs";
+import { getStorageFolder, getStorageServerConfigOptional } from "./serverConfig";
+import { getOwnThreadId } from "../../misc/https/certs";
+
+// The storage server's operation log: every mutation (never the data itself - just what happened, to what, when, how big, and who asked), every error, and the synchronization key points, streamed as JSON lines into <storage folder>/logs (see StreamingLogs for the file lifecycle). Everything here is a safe no-op on processes that have no storage folder - logging must never be the thing that breaks a request.
+
+export const LOGS_FOLDER_NAME = "logs";
+
+const getLogs = lazy((): StreamingLogs | undefined => {
+    let folder: string;
+    try {
+        folder = path.join(getStorageFolder(), LOGS_FOLDER_NAME);
+    } catch {
+        return undefined;
+    }
+    let threadId: string | undefined;
+    try {
+        threadId = getOwnThreadId(getStorageServerConfigOptional()?.rootDomain || "");
+    } catch {
+        threadId = undefined;
+    }
+    hookErrorLogging();
+    return new StreamingLogs({ folder, threadId });
+});
+
+function firstExternalIPv4(): string | undefined {
+    for (let addresses of Object.values(os.networkInterfaces())) {
+        for (let address of addresses || []) {
+            if (address.family === "IPv4" && !address.internal) return address.address;
+        }
+    }
+    return undefined;
+}
+
+// Computed once: none of it changes over a process's lifetime
+const baseFields = lazy(() => ({
+    pid: process.pid,
+    entryPoint: process.argv[1],
+    ip: firstExternalIPv4(),
+    domain: getStorageServerConfigOptional()?.domain,
+}));
+
+function write(kind: string, entry: { [key: string]: unknown }): void {
+    let logs = getLogs();
+    if (!logs) return;
+    logs.log({ kind, time: Date.now(), ...baseFields(), ...entry });
+}
+
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. */
+export function logMutation(entry: { op: string; account: string; bucketName: string; store?: string; path: string; toPath?: string; size?: number; writeTime?: number; callerId?: string; internal?: boolean }): void {
+    write("mutation", entry);
+}
+
+/** A synchronization key point: scans and full syncs starting/finishing, reconciles, boundary scans - what an operator greps for to see whether the fleet is converging. */
+export function logSyncEvent(entry: { event: string; store: string; source?: string; [key: string]: unknown }): void {
+    write("sync", entry);
+}
+
+export function logStorageError(message: string): void {
+    write("error", { message });
+}
+
+// Everything console.errored also lands in the log stream (with a reentrancy guard: a failure INSIDE logging must not log itself forever)
+let writingError = false;
+const hookErrorLogging = lazy(() => {
+    let original = console.error.bind(console);
+    console.error = (...args: unknown[]) => {
+        original(...args);
+        if (writingError) return;
+        writingError = true;
+        try {
+            logStorageError(args.map(x => typeof x === "string" && x || ((x as Error)?.stack ?? JSON.stringify(x))).join(" ").slice(0, 5000));
+        } catch { } finally {
+            writingError = false;
+        }
+    };
+});
+
+/** The log files this server holds - see StreamingLogs.listFiles. Empty on processes with no storage folder. */
+export async function listStorageLogFiles(): Promise<LogFileInfo[]> {
+    let logs = getLogs();
+    if (!logs) return [];
+    return await logs.listFiles();
+}
+
+/** One log file's bytes, always LZ4-compressed - see StreamingLogs.readFileCompressed (decode with decodeLogFile). */
+export async function readStorageLogFile(name: string): Promise<Buffer> {
+    let logs = getLogs();
+    if (!logs) {
+        throw new Error(`This process has no storage folder, so it has no logs to read`);
+    }
+    return await logs.readFileCompressed(name);
+}

package/storage/remoteStorage/storageServerState.ts

@@ -13,6 +13,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { StorageClientController } from "./storageClientController";
 import { isSelfSource } from "./storePlan";
 import { countBucketWrite, getBucketWriteStats, trackAccess, BucketWriteStats } from "./accessStats";
+import { logMutation } from "./storageLogs";
 
 /**
  * What this server HAS: the stores, by name (getStore) - made once, self-configuring from there -
@@ -43,7 +44,11 @@ export function getStore(account: string, bucketName: string, name: string, call
         createSource: config => createStoreSource({ sourceConfig: config.sourceConfig, folder, writeDelay: config.writeDelay }),
         applySource: (source, sourceConfig, writeDelay) => applySourceConfig(source, sourceConfig, writeDelay),
         onWriteCounted: (kind, bytes) => countBucketWrite(`${account}/${bucketName}`, kind, bytes),
-        onSyncTransfer: (operation, path, bytes) => trackAccess({ account, operation, path: `${bucketName}/${path}`, size: bytes }),
+        onSyncTransfer: (operation, path, bytes) => {
+            trackAccess({ account, operation, path: `${bucketName}/${path}`, size: bytes });
+            // Every synchronization write is logged per file, exactly like client mutations - the log is the full account of what moved
+            logMutation({ op: operation, account, bucketName, store: name, path, size: bytes });
+        },
         resolveSourceUrl: resolveSourceArchives,
         // The store is the one that knows when a config landed, and a config with upcoming windows is what boundary scans are armed from
         onRoutingApplied: routing => scheduleBoundaryWork(account, bucketName, routing),
@@ -105,6 +110,7 @@ export async function readBucketInternal(account: string, bucketName: string, co
 
 function aggregateArchivesConfig(bucketStores: BlobStore[], routing: RemoteConfig | undefined): ArchivesConfig {
     let index = { fileCount: 0, byteCount: 0 };
+    let markedIndex: { fileCount: number; byteCount: number; oldestDeleteTime?: number } = { fileCount: 0, byteCount: 0 };
     let indexSources: { debugName: string; fileCount: number; byteCount: number }[] = [];
     let syncing: SyncActivity[] = [];
     let readerDiskLimit: number | undefined;
@@ -112,6 +118,11 @@ function aggregateArchivesConfig(bucketStores: BlobStore[], routing: RemoteConfi
         let progress = store.getSyncProgress();
         index.fileCount += progress.index.fileCount;
         index.byteCount += progress.index.byteCount;
+        markedIndex.fileCount += progress.marked.fileCount;
+        markedIndex.byteCount += progress.marked.byteCount;
+        if (progress.marked.oldestDeleteTime !== undefined && (markedIndex.oldestDeleteTime === undefined || progress.marked.oldestDeleteTime < markedIndex.oldestDeleteTime)) {
+            markedIndex.oldestDeleteTime = progress.marked.oldestDeleteTime;
+        }
         indexSources.push(...progress.sources);
         syncing.push(...progress.syncing);
         readerDiskLimit = readerDiskLimit || progress.readerDiskLimit;
@@ -121,6 +132,7 @@ function aggregateArchivesConfig(bucketStores: BlobStore[], routing: RemoteConfi
         supportsChangesAfter: true,
         remoteConfig: routing,
         index,
+        markedIndex,
         indexSources,
         readerDiskLimit,
         syncing,
@@ -183,7 +195,9 @@ export async function writeRoutingConfig(account: string, bucketName: string, na
     let incoming = parseRoutingData(data);
     let current = await readRoutingFromDisk(account, bucketName);
     let stored = reinjectIntermediates(current, incoming);
-    await store.set({ path: ROUTING_FILE, data: Buffer.from(serializeRemoteConfig(stored)), lastModified: config?.lastModified });
+    let storedData = Buffer.from(serializeRemoteConfig(stored));
+    await store.set({ path: ROUTING_FILE, data: storedData, lastModified: config?.lastModified });
+    logMutation({ op: "routingConfig", account, bucketName, store: name, path: ROUTING_FILE, size: storedData.length, writeTime: config?.lastModified });
     broadcastRoutingChanged();
 }
 

package/storage/remoteStorage/storeSync.d.ts

@@ -50,4 +50,5 @@ export declare class StoreSync {
     private copySourceFiles;
     private enforceDiskLimit;
     private cleanupTombstones;
+    private enforceHistoryLimit;
 }

package/storage/remoteStorage/storeSync.ts

@@ -9,6 +9,8 @@ import { copyArchiveFile } from "../archiveHelpers";
 import { ArchivesBackblaze } from "../backblaze";
 import { ROUTING_FILE, getRoute, routeContains, parseRoutingData, getConfigVersion } from "./remoteConfig";
 import type { BlobStore, IndexEntry } from "./blobStore";
+import { getHistoryFactor, HISTORY_MIN_BYTES } from "./blobStore";
+import { logSyncEvent } from "./storageLogs";
 import { magenta } from "socket-function/src/formatting/logColors";
 
 // Everything that keeps a store's index in agreement with its sources, plus the two maintenance loops that follow from holding an index (evicting a bounded disk cache, expiring tombstones). The store owns the sources and decides where writes go; this decides what is scanned, pulled, pushed, evicted, and forgotten - so the write path stays readable without the several hundred lines of synchronization machinery interleaved into it.
@@ -100,6 +102,7 @@ export class StoreSync {
         }
         runInfinitePoll(CONFIG_POLL_INTERVAL, () => this.pollRoutingConfig(), this.store.stopped);
         runInfinitePoll(TOMBSTONE_CLEANUP_INTERVAL, () => this.cleanupTombstones(), this.store.stopped);
+        runInfinitePoll(TOMBSTONE_CLEANUP_INTERVAL, () => this.enforceHistoryLimit(), this.store.stopped);
         // Read live: the limit comes from the config, so a store that gains one later starts enforcing it without being restarted
         runInfinitePoll(DISK_LIMIT_CHECK_INTERVAL, () => this.enforceDiskLimit(), this.store.stopped);
     }
@@ -258,6 +261,7 @@ export class StoreSync {
             }
         }
         console.log(`Boundary scan of ${source.getDebugName()} finished in ${Math.round((Date.now() - scanStart) / 1000)}s (store ${this.store.folder}): ${changes.length} changes: ${formatScanTally(tally, changes.length)}`);
+        logSyncEvent({ event: "boundaryScanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, since: config.since, changes: changes.length, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone });
     }
 
     // ── per-source loops ──
@@ -326,6 +330,7 @@ export class StoreSync {
         let activity: SyncActivity = { type: "metadataScan", sourceDebugName: source.getDebugName(), startTime: scanStart };
         this.activities.add(activity);
         console.log(`Metadata scan of ${source.getDebugName()} starting (store ${this.store.folder})`);
+        logSyncEvent({ event: "scanStart", store: this.store.folder, source: source.getDebugName() });
         let progressTimer = setInterval(() => {
             console.log(`Metadata scan of ${source.getDebugName()} still running (${Math.round((Date.now() - scanStart) / 1000)}s, store ${this.store.folder})`);
         }, SYNC_PROGRESS_LOG_INTERVAL);
@@ -391,6 +396,7 @@ export class StoreSync {
         let union = indexSizeBefore + newPaths;
         let pct = (n: number) => `${Math.round(n / Math.max(union, 1) * 1000) / 10}%`;
         console.log(`Metadata scan of ${source.getDebugName()} finished in ${Math.round((Date.now() - scanStart) / 1000)}s (store ${this.store.folder}): ${files.length} listed vs ${indexSizeBefore} indexed (union ${union}): ${formatScanTally(tally, union)}, ${missingOnSource} in index but missing on source (${pct(missingOnSource)}), ${removedFromIndex} removed from index (${pct(removedFromIndex)})`);
+        logSyncEvent({ event: "scanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, listed: files.length, indexedBefore: indexSizeBefore, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone, missingOnSource, removedFromIndex });
         state.changesAfterTime = Math.max(state.changesAfterTime, scanStart - CHANGES_POLL_OVERLAP);
         return seen;
     }
@@ -459,6 +465,9 @@ export class StoreSync {
         } else if (pushed) {
             console.log(`Reconciled sync source ${source.getDebugName()} (store ${this.store.folder}): pushed ${pushed} files it was missing or held older copies of`);
         }
+        if (pushed || failed) {
+            logSyncEvent({ event: "reconcileFinish", store: this.store.folder, source: source.getDebugName(), pushed, failed, aborted });
+        }
     }
 
     private updateScanIndex(sourceIndex: number, file: ArchiveFileInfo): ScanOutcome {
@@ -526,6 +535,7 @@ export class StoreSync {
             totalBytes,
         };
         this.activities.add(activity);
+        logSyncEvent({ event: "fullSyncStart", store: this.store.folder, source: source.getDebugName(), files: pending.length, bytes: totalBytes });
         let progressLogged = false;
         let logProgress = () => {
             progressLogged = true;
@@ -602,6 +612,7 @@ export class StoreSync {
             if (progressLogged) {
                 logProgress();
             }
+            logSyncEvent({ event: "fullSyncFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - activity.startTime, copiedFiles: activity.doneFiles, copiedBytes: activity.doneBytes, totalFiles: pending.length, totalBytes });
         }
     }
 
@@ -665,6 +676,8 @@ export class StoreSync {
         for (let [key, tombstone] of this.store.deletedEntries()) {
             if (this.store.stopped.stop) return;
             if (tombstone.writeTime > cutoff) continue;
+            // A MARKED deletion never expires by time - its bytes are still on disk, and purging the tombstone would let the next disk scan resurrect them as live. Retention expires it by SIZE (enforceHistoryLimit); only once the bytes are gone does the plain tombstone age out here.
+            if (this.store.getMarkedEntry(key)) continue;
             this.store.purgeIndexEntry(key);
             for (let i = 0; i < this.store.sources.length; i++) {
                 if (!this.isLive(i)) continue;
@@ -678,4 +691,29 @@ export class StoreSync {
             }
         }
     }
+
+    // Deletion history retention: marked files keep their bytes on our disk until the history outgrows max(HISTORY_MIN_BYTES, live bytes * history factor); then the OLDEST deletions lose their bytes and become plain tombstones, which age out normally (cleanupTombstones)
+    private async enforceHistoryLimit(): Promise<void> {
+        let allowed = Math.max(HISTORY_MIN_BYTES, this.store.indexTotals().byteCount * await getHistoryFactor());
+        let marked: { key: string; size: number; deleteTime: number }[] = [];
+        let totalBytes = 0;
+        for (let [key, entry] of this.store.markedEntries()) {
+            marked.push({ key, size: entry.size, deleteTime: entry.deleteTime });
+            totalBytes += entry.size;
+        }
+        if (totalBytes <= allowed) return;
+        let startedWithBytes = totalBytes;
+        sort(marked, x => x.deleteTime);
+        let dropped = 0;
+        let droppedBytes = 0;
+        for (let { key, size } of marked) {
+            if (this.store.stopped.stop) return;
+            if (totalBytes <= allowed) break;
+            await this.store.dropMarkedHistory(key);
+            totalBytes -= size;
+            dropped++;
+            droppedBytes += size;
+        }
+        console.log(`Deletion history of store ${this.store.folder} outgrew its budget (${formatNumber(startedWithBytes)}B kept vs ${formatNumber(allowed)}B allowed): dropped the ${dropped} oldest marked files (${formatNumber(droppedBytes)}B); their tombstones remain until they age out`);
+    }
 }

package/yarn.lock

@@ -1937,10 +1937,10 @@ slash@^3.0.0:
   resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
   integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
 
-socket-function@^1.2.30:
-  version "1.2.30"
-  resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.2.30.tgz#95979b29a322c7e8d184ce7219310820d00cd042"
-  integrity sha512-A69c6ejRICUQXjC/3S3S2cd7CpoUjR1jjenPIEMA5b6G7xfUdQQqVM51hvxH73qDY/QhGV2UyZTDmlmgPh+f+g==
+socket-function@^1.2.31:
+  version "1.2.31"
+  resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.2.31.tgz#b9ced45d5c99c310ecbf775d13a731e2aefff8e2"
+  integrity sha512-TBAHfSS2iapdweD6sOt2wRvynlrZ9B9EYSYBpm1x84hgbn/xWJRu1yL3e4uHeC07Fun4l8+meUEM6nZkujls0A==
   dependencies:
     "@types/pako" "^2.0.3"
     "@types/ws" "^8.5.3"