sliftutils 1.7.101 → 1.7.103

package/storage/remoteStorage/spec.md

@@ -6,6 +6,10 @@ This is the reasoning behind the remote storage system (createArchives / storage
 
 Every store fully rescans its sources' metadata (on startup and periodically), and every write — including deletions — is ordered by last-write time. Deletions are tombstones: an empty file IS a missing file, kept as a size-0 index entry (for a week) so the deletion itself propagates and reconciles like any other write. Because everything is a timestamped write and scans are bidirectional (pull what they have, push what they're missing), any two sources can be merged in any order and converge to the same state: newest write time wins, per file. There is no operation whose loss corrupts the system — a failed background write, a missed delete, a source that was down for a day — the next scan reconciles it.
 
+## Deletions are marked, not performed — the bytes become history
+
+A deletion never removes bytes from a server's disk: the index MARKS the key deleted (keeping its entry - size, holder, original write time), reads and listings stop finding it, and the deletion propagates as before. The bytes sit on disk as deletion history, per node, until the history outgrows max(10GB, live bytes × history factor — see getHistoryFactor) — then the OLDEST deletions lose their bytes and demote to plain tombstones, which age out on the week timer as before (a marked deletion never expires by time, only by size: purging its tombstone while its bytes remain would let the next disk scan resurrect them). While the history holds a file, includeMarked reads and listings can still see it, and undelete (a set carrying SetConfig.undelete) flips the index entry back to live with a fresh write time — the bytes never moved, so reads just work again, and the restore propagates to peers the way the deletion did. Raw sources with no index (backblaze) keep the old behavior: deletions materialize as empty files there, so history lives on server disks only.
+
 ## storage/storagerouting.json Has the routing config and is duplicated on every node. 
 
 Each bucket stores its complete routing config (the full redundancy list) inside itself, at storage/storagerouting.json, on every source. Discovery is therefore trivial: as long as ONE node is up, a client reading it gets the full overview of the intended sources. And because clients re-discover on every startup (and re-read every 5 minutes), a developer can change the configuration in one place and clients rapidly accept it — no redeploy, no coordinated restart of the fleet.

package/storage/remoteStorage/storageController.d.ts

@@ -3,6 +3,7 @@
 import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, FindConfig, SourceConfig } from "../IArchives";
 import { ActiveBucketInfo, ServerBucketInfo } from "./storageServerState";
 import { AccessTotals, AccessSummaryState } from "./accessStats";
+import { LogFileInfo } from "../StreamingLogs";
 import type { SummaryEntry } from "../../treeSummary";
 export declare const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
 export declare const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
@@ -84,6 +85,7 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
         };
         internal?: boolean;
         includeTombstones?: boolean;
+        includeMarked?: boolean;
     }) => Promise<{
         data: Buffer;
         writeTime: number;
@@ -98,6 +100,7 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
         lastModified?: number;
         forceSetImmutable?: boolean;
         internal?: boolean;
+        undelete?: boolean;
     }) => Promise<void>;
     del: (config: {
         account: string;
@@ -182,6 +185,13 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
         account: string;
         bucketName: string;
     }) => Promise<ArchivesSyncStatus>;
+    listLogFiles: (config: {
+        account: string;
+    }) => Promise<LogFileInfo[]>;
+    getLogFile: (config: {
+        account: string;
+        name: string;
+    }) => Promise<Buffer>;
     startLargeFile: (config: {
         account: string;
         bucketName: string;

package/storage/remoteStorage/storageController.ts

@@ -20,6 +20,8 @@ import { BlobStore } from "./blobStore";
 import { getRoutingFileResult } from "./bucketDisk";
 import { StorageClientController } from "./storageClientController";
 import { trackAccess, trackAccessCall, getAccessTotals, readAccessSummaries, clearAccountAccessStats, AccessTotals, AccessSummaryState } from "./accessStats";
+import { logMutation, listStorageLogFiles, readStorageLogFile } from "./storageLogs";
+import { LogFileInfo } from "../StreamingLogs";
 import { assertValidName, assertValidPath, assertValidArgs } from "./validation";
 import type { SummaryEntry } from "../../treeSummary";
 
@@ -275,7 +277,7 @@ class RemoteStorageControllerBase {
 
     @assertValidArgs
     @trackAccessCall("get")
-    async get2(config: { account: string; bucketName: string; path: string; sourceConfig: SourceConfig; range?: { start: number; end: number }; internal?: boolean; includeTombstones?: boolean }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
+    async get2(config: { account: string; bucketName: string; path: string; sourceConfig: SourceConfig; range?: { start: number; end: number }; internal?: boolean; includeTombstones?: boolean; includeMarked?: boolean }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
         if (config.path === ROUTING_FILE) {
             // The routing file lives outside every store and even outside the bucket (it is what CREATES it), so it is read straight off the disk - absent means undefined, exactly like any file read
             return await getRoutingFileResult(config.account, config.bucketName);
@@ -287,8 +289,9 @@ class RemoteStorageControllerBase {
     }
     @assertValidArgs
     @trackAccessCall("set")
-    async set(config: { account: string; bucketName: string; path: string; data: Buffer; sourceConfig: SourceConfig; lastModified?: number; forceSetImmutable?: boolean; internal?: boolean }): Promise<void> {
+    async set(config: { account: string; bucketName: string; path: string; data: Buffer; sourceConfig: SourceConfig; lastModified?: number; forceSetImmutable?: boolean; internal?: boolean; undelete?: boolean }): Promise<void> {
         assertWritesAllowed();
+        let caller = callerId();
         // Copied because the wire hands us a plain Uint8Array view, not a real Buffer
         let data = Buffer.from(config.data);
         if (config.path === ROUTING_FILE) {
@@ -296,21 +299,26 @@ class RemoteStorageControllerBase {
             return await writeRoutingConfig(config.account, config.bucketName, config.sourceConfig.name, data, config);
         }
         await withStore(config, store => store.set({ ...config, data }));
+        logMutation({ op: config.undelete && "undelete" || "set", account: config.account, bucketName: config.bucketName, store: config.sourceConfig.name, path: config.path, size: data.length, writeTime: config.lastModified, callerId: caller, internal: config.internal });
     }
     @assertValidArgs
     @trackAccessCall("del")
     async del(config: { account: string; bucketName: string; path: string; sourceConfig: SourceConfig; lastModified?: number; internal?: boolean }): Promise<void> {
         assertWritesAllowed();
+        let caller = callerId();
         await withStore(config, store => store.del(config));
+        logMutation({ op: "del", account: config.account, bucketName: config.bucketName, store: config.sourceConfig.name, path: config.path, writeTime: config.lastModified, callerId: caller, internal: config.internal });
     }
     @assertValidArgs
     @trackAccessCall("move")
     async move(config: { account: string; bucketName: string; fromPath: string; toPath: string; sourceConfig: SourceConfig }): Promise<void> {
         assertWritesAllowed();
+        let caller = callerId();
         // assertValidArgs only knows the well-known `path` field, so the two paths of a move are validated here
         assertValidPath(config.fromPath);
         assertValidPath(config.toPath);
         await withStore(config, store => store.move(config));
+        logMutation({ op: "move", account: config.account, bucketName: config.bucketName, store: config.sourceConfig.name, path: config.fromPath, toPath: config.toPath, callerId: caller });
     }
     @assertValidArgs
     @trackAccessCall("getInfo")
@@ -380,6 +388,17 @@ class RemoteStorageControllerBase {
         return await bucketSyncStatus(config.account, config.bucketName);
     }
 
+    /** The operation-log files THIS server holds (every server logs its own operations - ask each one; see listAllServerLogFiles in createArchives). */
+    @assertValidArgs
+    async listLogFiles(config: { account: string }): Promise<LogFileInfo[]> {
+        return await listStorageLogFiles();
+    }
+    /** One log file's bytes, always LZ4-compressed (a live file is flushed and compressed in memory before sending). Decode with decodeLogFile. */
+    @assertValidArgs
+    async getLogFile(config: { account: string; name: string }): Promise<Buffer> {
+        return await readStorageLogFile(config.name);
+    }
+
     @assertValidArgs
     async startLargeFile(config: { account: string; bucketName: string; path: string; sourceConfig: SourceConfig; lastModified?: number; forceSetImmutable?: boolean; noChecks?: boolean; internal?: boolean }): Promise<string> {
         assertWritesAllowed();
@@ -404,7 +423,11 @@ class RemoteStorageControllerBase {
         let info = largeUploadInfo.get(config.uploadId);
         if (!info) throw new Error(`Unknown large upload ${config.uploadId}`);
         largeUploadInfo.delete(config.uploadId);
-        await getStore(info.account, info.bucketName, info.storeName).finishLargeUpload({ id: config.uploadId, path: info.path, lastModified: info.lastModified, forceSetImmutable: info.forceSetImmutable, noChecks: info.noChecks, internal: info.internal });
+        let caller = callerId();
+        let store = getStore(info.account, info.bucketName, info.storeName);
+        await store.finishLargeUpload({ id: config.uploadId, path: info.path, lastModified: info.lastModified, forceSetImmutable: info.forceSetImmutable, noChecks: info.noChecks, internal: info.internal });
+        let written = await store.getInfo({ path: info.path });
+        logMutation({ op: "setLarge", account: info.account, bucketName: info.bucketName, store: info.storeName, path: info.path, size: written?.size, writeTime: written?.writeTime, callerId: caller });
     }
     /** Best-effort cleanup: an unknown upload has nothing left to cancel. */
     async cancelLargeFile(config: { uploadId: string }): Promise<void> {
@@ -527,6 +550,15 @@ async function withStore<T>(config: { account: string; bucketName: string; sourc
     return await fn(findBucketStore(config.account, config.bucketName, config.sourceConfig));
 }
 
+// The caller's node id for the mutation log - read synchronously at method entry (the call context does not survive awaits), and tolerant of there being none (local calls)
+function callerId(): string | undefined {
+    try {
+        return SocketFunction.getCaller()?.nodeId;
+    } catch {
+        return undefined;
+    }
+}
+
 const largeUploadInfo = new Map<string, { account: string; bucketName: string; path: string; lastModified?: number; forceSetImmutable?: boolean; noChecks?: boolean; internal?: boolean; storeName: string }>();
 
 const accountAccess: SocketFunctionHook = async (context) => {
@@ -577,6 +609,8 @@ export const RemoteStorageController = SocketFunction.register(
         getAccessStats: { hooks: [accountAccess] },
         getAccessSummaries: { hooks: [accountAccess] },
         getSyncStatus: { hooks: [accountAccess] },
+        listLogFiles: { hooks: [accountAccess] },
+        getLogFile: { hooks: [accountAccess] },
         startLargeFile: { hooks: [accountAccess] },
         uploadPart: { hooks: [uploadAccess] },
         finishLargeFile: { hooks: [uploadAccess] },

package/storage/remoteStorage/storageLogs.d.ts

@@ -0,0 +1,29 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { LogFileInfo } from "../StreamingLogs";
+export declare const LOGS_FOLDER_NAME = "logs";
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. */
+export declare function logMutation(entry: {
+    op: string;
+    account: string;
+    bucketName: string;
+    store?: string;
+    path: string;
+    toPath?: string;
+    size?: number;
+    writeTime?: number;
+    callerId?: string;
+    internal?: boolean;
+}): void;
+/** A synchronization key point: scans and full syncs starting/finishing, reconciles, boundary scans - what an operator greps for to see whether the fleet is converging. */
+export declare function logSyncEvent(entry: {
+    event: string;
+    store: string;
+    source?: string;
+    [key: string]: unknown;
+}): void;
+export declare function logStorageError(message: string): void;
+/** The log files this server holds - see StreamingLogs.listFiles. Empty on processes with no storage folder. */
+export declare function listStorageLogFiles(): Promise<LogFileInfo[]>;
+/** One log file's bytes, always LZ4-compressed - see StreamingLogs.readFileCompressed (decode with decodeLogFile). */
+export declare function readStorageLogFile(name: string): Promise<Buffer>;

package/storage/remoteStorage/storageLogs.ts

@@ -0,0 +1,96 @@
+import os from "os";
+import path from "path";
+import { lazy } from "socket-function/src/caching";
+import { StreamingLogs, LogFileInfo } from "../StreamingLogs";
+import { getStorageFolder, getStorageServerConfigOptional } from "./serverConfig";
+import { getOwnThreadId } from "../../misc/https/certs";
+
+// The storage server's operation log: every mutation (never the data itself - just what happened, to what, when, how big, and who asked), every error, and the synchronization key points, streamed as JSON lines into <storage folder>/logs (see StreamingLogs for the file lifecycle). Everything here is a safe no-op on processes that have no storage folder - logging must never be the thing that breaks a request.
+
+export const LOGS_FOLDER_NAME = "logs";
+
+const getLogs = lazy((): StreamingLogs | undefined => {
+    let folder: string;
+    try {
+        folder = path.join(getStorageFolder(), LOGS_FOLDER_NAME);
+    } catch {
+        return undefined;
+    }
+    let threadId: string | undefined;
+    try {
+        threadId = getOwnThreadId(getStorageServerConfigOptional()?.rootDomain || "");
+    } catch {
+        threadId = undefined;
+    }
+    hookErrorLogging();
+    return new StreamingLogs({ folder, threadId });
+});
+
+function firstExternalIPv4(): string | undefined {
+    for (let addresses of Object.values(os.networkInterfaces())) {
+        for (let address of addresses || []) {
+            if (address.family === "IPv4" && !address.internal) return address.address;
+        }
+    }
+    return undefined;
+}
+
+// Computed once: none of it changes over a process's lifetime
+const baseFields = lazy(() => ({
+    pid: process.pid,
+    entryPoint: process.argv[1],
+    ip: firstExternalIPv4(),
+    domain: getStorageServerConfigOptional()?.domain,
+}));
+
+function write(kind: string, entry: { [key: string]: unknown }): void {
+    let logs = getLogs();
+    if (!logs) return;
+    logs.log({ kind, time: Date.now(), ...baseFields(), ...entry });
+}
+
+/** One mutation the server performed: set/del/move/undelete/setLarge/routingConfig, plus the per-file synchronization writes ("sync get"/"sync set"). Sizes and times, never the data. internal marks writes pushed by a peer's synchronization rather than a client. */
+export function logMutation(entry: { op: string; account: string; bucketName: string; store?: string; path: string; toPath?: string; size?: number; writeTime?: number; callerId?: string; internal?: boolean }): void {
+    write("mutation", entry);
+}
+
+/** A synchronization key point: scans and full syncs starting/finishing, reconciles, boundary scans - what an operator greps for to see whether the fleet is converging. */
+export function logSyncEvent(entry: { event: string; store: string; source?: string; [key: string]: unknown }): void {
+    write("sync", entry);
+}
+
+export function logStorageError(message: string): void {
+    write("error", { message });
+}
+
+// Everything console.errored also lands in the log stream (with a reentrancy guard: a failure INSIDE logging must not log itself forever)
+let writingError = false;
+const hookErrorLogging = lazy(() => {
+    let original = console.error.bind(console);
+    console.error = (...args: unknown[]) => {
+        original(...args);
+        if (writingError) return;
+        writingError = true;
+        try {
+            logStorageError(args.map(x => typeof x === "string" && x || ((x as Error)?.stack ?? JSON.stringify(x))).join(" ").slice(0, 5000));
+        } catch { } finally {
+            writingError = false;
+        }
+    };
+});
+
+/** The log files this server holds - see StreamingLogs.listFiles. Empty on processes with no storage folder. */
+export async function listStorageLogFiles(): Promise<LogFileInfo[]> {
+    let logs = getLogs();
+    if (!logs) return [];
+    return await logs.listFiles();
+}
+
+/** One log file's bytes, always LZ4-compressed - see StreamingLogs.readFileCompressed (decode with decodeLogFile). */
+export async function readStorageLogFile(name: string): Promise<Buffer> {
+    let logs = getLogs();
+    if (!logs) {
+        throw new Error(`This process has no storage folder, so it has no logs to read`);
+    }
+    return await logs.readFileCompressed(name);
+}

package/storage/remoteStorage/storageServerState.ts

@@ -12,7 +12,8 @@ import { scheduleBoundaryWork, reinjectIntermediates } from "./intermediateManag
 import { SocketFunction } from "socket-function/SocketFunction";
 import { StorageClientController } from "./storageClientController";
 import { isSelfSource } from "./storePlan";
-import { countBucketWrite, getBucketWriteStats, BucketWriteStats } from "./accessStats";
+import { countBucketWrite, getBucketWriteStats, trackAccess, BucketWriteStats } from "./accessStats";
+import { logMutation } from "./storageLogs";
 
 /**
  * What this server HAS: the stores, by name (getStore) - made once, self-configuring from there -
@@ -43,6 +44,11 @@ export function getStore(account: string, bucketName: string, name: string, call
         createSource: config => createStoreSource({ sourceConfig: config.sourceConfig, folder, writeDelay: config.writeDelay }),
         applySource: (source, sourceConfig, writeDelay) => applySourceConfig(source, sourceConfig, writeDelay),
         onWriteCounted: (kind, bytes) => countBucketWrite(`${account}/${bucketName}`, kind, bytes),
+        onSyncTransfer: (operation, path, bytes) => {
+            trackAccess({ account, operation, path: `${bucketName}/${path}`, size: bytes });
+            // Every synchronization write is logged per file, exactly like client mutations - the log is the full account of what moved
+            logMutation({ op: operation, account, bucketName, store: name, path, size: bytes });
+        },
         resolveSourceUrl: resolveSourceArchives,
         // The store is the one that knows when a config landed, and a config with upcoming windows is what boundary scans are armed from
         onRoutingApplied: routing => scheduleBoundaryWork(account, bucketName, routing),
@@ -104,6 +110,7 @@ export async function readBucketInternal(account: string, bucketName: string, co
 
 function aggregateArchivesConfig(bucketStores: BlobStore[], routing: RemoteConfig | undefined): ArchivesConfig {
     let index = { fileCount: 0, byteCount: 0 };
+    let markedIndex: { fileCount: number; byteCount: number; oldestDeleteTime?: number } = { fileCount: 0, byteCount: 0 };
     let indexSources: { debugName: string; fileCount: number; byteCount: number }[] = [];
     let syncing: SyncActivity[] = [];
     let readerDiskLimit: number | undefined;
@@ -111,6 +118,11 @@ function aggregateArchivesConfig(bucketStores: BlobStore[], routing: RemoteConfi
         let progress = store.getSyncProgress();
         index.fileCount += progress.index.fileCount;
         index.byteCount += progress.index.byteCount;
+        markedIndex.fileCount += progress.marked.fileCount;
+        markedIndex.byteCount += progress.marked.byteCount;
+        if (progress.marked.oldestDeleteTime !== undefined && (markedIndex.oldestDeleteTime === undefined || progress.marked.oldestDeleteTime < markedIndex.oldestDeleteTime)) {
+            markedIndex.oldestDeleteTime = progress.marked.oldestDeleteTime;
+        }
         indexSources.push(...progress.sources);
         syncing.push(...progress.syncing);
         readerDiskLimit = readerDiskLimit || progress.readerDiskLimit;
@@ -120,6 +132,7 @@ function aggregateArchivesConfig(bucketStores: BlobStore[], routing: RemoteConfi
         supportsChangesAfter: true,
         remoteConfig: routing,
         index,
+        markedIndex,
         indexSources,
         readerDiskLimit,
         syncing,
@@ -182,7 +195,9 @@ export async function writeRoutingConfig(account: string, bucketName: string, na
     let incoming = parseRoutingData(data);
     let current = await readRoutingFromDisk(account, bucketName);
     let stored = reinjectIntermediates(current, incoming);
-    await store.set({ path: ROUTING_FILE, data: Buffer.from(serializeRemoteConfig(stored)), lastModified: config?.lastModified });
+    let storedData = Buffer.from(serializeRemoteConfig(stored));
+    await store.set({ path: ROUTING_FILE, data: storedData, lastModified: config?.lastModified });
+    logMutation({ op: "routingConfig", account, bucketName, store: name, path: ROUTING_FILE, size: storedData.length, writeTime: config?.lastModified });
     broadcastRoutingChanged();
 }
 

package/storage/remoteStorage/storeSync.d.ts

@@ -50,4 +50,5 @@ export declare class StoreSync {
     private copySourceFiles;
     private enforceDiskLimit;
     private cleanupTombstones;
+    private enforceHistoryLimit;
 }

package/storage/remoteStorage/storeSync.ts

@@ -9,6 +9,8 @@ import { copyArchiveFile } from "../archiveHelpers";
 import { ArchivesBackblaze } from "../backblaze";
 import { ROUTING_FILE, getRoute, routeContains, parseRoutingData, getConfigVersion } from "./remoteConfig";
 import type { BlobStore, IndexEntry } from "./blobStore";
+import { getHistoryFactor, HISTORY_MIN_BYTES } from "./blobStore";
+import { logSyncEvent } from "./storageLogs";
 import { magenta } from "socket-function/src/formatting/logColors";
 
 // Everything that keeps a store's index in agreement with its sources, plus the two maintenance loops that follow from holding an index (evicting a bounded disk cache, expiring tombstones). The store owns the sources and decides where writes go; this decides what is scanned, pulled, pushed, evicted, and forgotten - so the write path stays readable without the several hundred lines of synchronization machinery interleaved into it.
@@ -100,6 +102,7 @@ export class StoreSync {
         }
         runInfinitePoll(CONFIG_POLL_INTERVAL, () => this.pollRoutingConfig(), this.store.stopped);
         runInfinitePoll(TOMBSTONE_CLEANUP_INTERVAL, () => this.cleanupTombstones(), this.store.stopped);
+        runInfinitePoll(TOMBSTONE_CLEANUP_INTERVAL, () => this.enforceHistoryLimit(), this.store.stopped);
         // Read live: the limit comes from the config, so a store that gains one later starts enforcing it without being restarted
         runInfinitePoll(DISK_LIMIT_CHECK_INTERVAL, () => this.enforceDiskLimit(), this.store.stopped);
     }
@@ -244,6 +247,7 @@ export class StoreSync {
             }
             let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: file.path, size: file.size, writeTime: file.createTime, forceSetImmutable: true, noChecks: true, internal: true });
             if (!copied) continue;
+            this.store.noteSyncTransfer("sync get", file.path, copied.size);
             if (copied.size === 0) {
                 this.store.setIndexDeleted(file.path, copied.writeTime);
                 tally.tombstone++;
@@ -257,6 +261,7 @@ export class StoreSync {
             }
         }
         console.log(`Boundary scan of ${source.getDebugName()} finished in ${Math.round((Date.now() - scanStart) / 1000)}s (store ${this.store.folder}): ${changes.length} changes: ${formatScanTally(tally, changes.length)}`);
+        logSyncEvent({ event: "boundaryScanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, since: config.since, changes: changes.length, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone });
     }
 
     // ── per-source loops ──
@@ -325,6 +330,7 @@ export class StoreSync {
         let activity: SyncActivity = { type: "metadataScan", sourceDebugName: source.getDebugName(), startTime: scanStart };
         this.activities.add(activity);
         console.log(`Metadata scan of ${source.getDebugName()} starting (store ${this.store.folder})`);
+        logSyncEvent({ event: "scanStart", store: this.store.folder, source: source.getDebugName() });
         let progressTimer = setInterval(() => {
             console.log(`Metadata scan of ${source.getDebugName()} still running (${Math.round((Date.now() - scanStart) / 1000)}s, store ${this.store.folder})`);
         }, SYNC_PROGRESS_LOG_INTERVAL);
@@ -390,6 +396,7 @@ export class StoreSync {
         let union = indexSizeBefore + newPaths;
         let pct = (n: number) => `${Math.round(n / Math.max(union, 1) * 1000) / 10}%`;
         console.log(`Metadata scan of ${source.getDebugName()} finished in ${Math.round((Date.now() - scanStart) / 1000)}s (store ${this.store.folder}): ${files.length} listed vs ${indexSizeBefore} indexed (union ${union}): ${formatScanTally(tally, union)}, ${missingOnSource} in index but missing on source (${pct(missingOnSource)}), ${removedFromIndex} removed from index (${pct(removedFromIndex)})`);
+        logSyncEvent({ event: "scanFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - scanStart, listed: files.length, indexedBefore: indexSizeBefore, newPaths: tally.new, updated: tally.updated, tombstones: tally.tombstone, missingOnSource, removedFromIndex });
         state.changesAfterTime = Math.max(state.changesAfterTime, scanStart - CHANGES_POLL_OVERLAP);
         return seen;
     }
@@ -427,6 +434,7 @@ export class StoreSync {
                     // A deletion only needs pushing while the source still holds an older copy. It travels as del (never as an empty set - set rejects empty buffers), with the ORIGINAL deletion time so ordering survives.
                     if (theirTime === undefined) continue;
                     await source.del(key, { lastModified: writeTime, noChecks: true, internal: true });
+                    this.store.noteSyncTransfer("sync set", key, 0);
                     pushed++;
                     consecutiveFailures = 0;
                     continue;
@@ -435,6 +443,7 @@ export class StoreSync {
                 if (!holder) continue;
                 let copied = await copyArchiveFile({ from: holder, to: source, path: key, size: entry.size, writeTime: entry.writeTime, forceSetImmutable: true, noChecks: true, internal: true });
                 if (!copied) continue;
+                this.store.noteSyncTransfer("sync set", key, copied.size);
                 pushed++;
                 consecutiveFailures = 0;
             } catch (e) {
@@ -456,6 +465,9 @@ export class StoreSync {
         } else if (pushed) {
             console.log(`Reconciled sync source ${source.getDebugName()} (store ${this.store.folder}): pushed ${pushed} files it was missing or held older copies of`);
         }
+        if (pushed || failed) {
+            logSyncEvent({ event: "reconcileFinish", store: this.store.folder, source: source.getDebugName(), pushed, failed, aborted });
+        }
     }
 
     private updateScanIndex(sourceIndex: number, file: ArchiveFileInfo): ScanOutcome {
@@ -523,6 +535,7 @@ export class StoreSync {
             totalBytes,
         };
         this.activities.add(activity);
+        logSyncEvent({ event: "fullSyncStart", store: this.store.folder, source: source.getDebugName(), files: pending.length, bytes: totalBytes });
         let progressLogged = false;
         let logProgress = () => {
             progressLogged = true;
@@ -558,6 +571,7 @@ export class StoreSync {
                     try {
                         let copied = await copyArchiveFile({ from: source, to: this.store.sources[0].source, path: key, size: entry.size, writeTime: entry.writeTime, forceSetImmutable: true, noChecks: true, internal: true });
                         if (copied) {
+                            this.store.noteSyncTransfer("sync get", key, copied.size);
                             // Only move the entry's source if it wasn't changed while we copied
                             if (this.entryUnchanged(key, entry)) {
                                 this.store.setIndexEntry(key, { writeTime: copied.writeTime, size: copied.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
@@ -598,6 +612,7 @@ export class StoreSync {
             if (progressLogged) {
                 logProgress();
             }
+            logSyncEvent({ event: "fullSyncFinish", store: this.store.folder, source: source.getDebugName(), durationMs: Date.now() - activity.startTime, copiedFiles: activity.doneFiles, copiedBytes: activity.doneBytes, totalFiles: pending.length, totalBytes });
         }
     }
 
@@ -661,6 +676,8 @@ export class StoreSync {
         for (let [key, tombstone] of this.store.deletedEntries()) {
             if (this.store.stopped.stop) return;
             if (tombstone.writeTime > cutoff) continue;
+            // A MARKED deletion never expires by time - its bytes are still on disk, and purging the tombstone would let the next disk scan resurrect them as live. Retention expires it by SIZE (enforceHistoryLimit); only once the bytes are gone does the plain tombstone age out here.
+            if (this.store.getMarkedEntry(key)) continue;
             this.store.purgeIndexEntry(key);
             for (let i = 0; i < this.store.sources.length; i++) {
                 if (!this.isLive(i)) continue;
@@ -674,4 +691,29 @@ export class StoreSync {
             }
         }
     }
+
+    // Deletion history retention: marked files keep their bytes on our disk until the history outgrows max(HISTORY_MIN_BYTES, live bytes * history factor); then the OLDEST deletions lose their bytes and become plain tombstones, which age out normally (cleanupTombstones)
+    private async enforceHistoryLimit(): Promise<void> {
+        let allowed = Math.max(HISTORY_MIN_BYTES, this.store.indexTotals().byteCount * await getHistoryFactor());
+        let marked: { key: string; size: number; deleteTime: number }[] = [];
+        let totalBytes = 0;
+        for (let [key, entry] of this.store.markedEntries()) {
+            marked.push({ key, size: entry.size, deleteTime: entry.deleteTime });
+            totalBytes += entry.size;
+        }
+        if (totalBytes <= allowed) return;
+        let startedWithBytes = totalBytes;
+        sort(marked, x => x.deleteTime);
+        let dropped = 0;
+        let droppedBytes = 0;
+        for (let { key, size } of marked) {
+            if (this.store.stopped.stop) return;
+            if (totalBytes <= allowed) break;
+            await this.store.dropMarkedHistory(key);
+            totalBytes -= size;
+            dropped++;
+            droppedBytes += size;
+        }
+        console.log(`Deletion history of store ${this.store.folder} outgrew its budget (${formatNumber(startedWithBytes)}B kept vs ${formatNumber(allowed)}B allowed): dropped the ${dropped} oldest marked files (${formatNumber(droppedBytes)}B); their tombstones remain until they age out`);
+    }
 }