sliftutils 1.7.102 → 1.7.104

package/storage/StreamingLogs.ts

@@ -0,0 +1,412 @@
+import fs from "fs";
+import path from "path";
+import { sort } from "socket-function/src/misc";
+import { LZ4 } from "socket-function/src/lz4/LZ4";
+
+// Generic append-only JSON-lines logging over a folder. Every process streams to its OWN file (named by pid + process start time + thread id, which together identify a process incarnation), rotating to a new file at a size limit and LZ4-compressing the finished one. Every process using the folder also MAINTAINS it on a randomized interval: dead processes' files are compressed for them, duplicate compressions are resolved, and the oldest files are deleted once the folder outgrows its byte budget. Instantiate one per folder and call log().
+
+// Appends are coalesced for this long; a crash loses at most this much
+const FLUSH_DELAY = 5000;
+// A file reaching this size is rotated out and compressed
+const MAX_FILE_BYTES = 100 * 1024 * 1024;
+// The folder's total budget; once exceeded, the oldest files are deleted
+const TOTAL_LIMIT_BYTES = 30 * 1024 * 1024 * 1024;
+// Maintenance runs at a random point in this range, so the processes sharing the folder drift apart instead of stepping on each other every pass
+const MAINTENANCE_MIN_INTERVAL = 30 * 60 * 1000;
+const MAINTENANCE_MAX_INTERVAL = 60 * 60 * 1000;
+// A pid is only the same PROCESS if its start time matches the file name's - within this much, because our own start time is derived from process.uptime and the OS's is read off /proc
+const PROCESS_START_TOLERANCE = 60 * 1000;
+// If flushing fails persistently (disk gone), pending entries are capped rather than growing forever
+const MAX_PENDING_ENTRIES = 10000;
+
+const UNCOMPRESSED_EXT = ".jsonl";
+const COMPRESSED_EXT = ".jsonl.lz4";
+
+/** Everything a log file's NAME says about it (plus its on-disk size). Times bound the entries roughly: from is when its process started writing it, to is when it was compressed. */
+export type LogFileInfo = {
+    name: string;
+    /** Bytes on disk (compressed bytes for compressed files) */
+    size: number;
+    pid: number;
+    processStartTime: number;
+    threadId: string;
+    /** When the process started writing this file */
+    startTime: number;
+    /** When the file was compressed - absent while it is still being streamed to */
+    endTime?: number;
+    /** How many entries it holds - only counted at compression, so absent on live files */
+    entryCount?: number;
+    compressed: boolean;
+    /** Whether the writing process is still alive (always false for compressed files) */
+    active: boolean;
+};
+
+type ParsedName = {
+    pid: number;
+    processStartTime: number;
+    threadId: string;
+    startTime: number;
+    endTime?: number;
+    entryCount?: number;
+    compressed: boolean;
+    /** The part identifying the ORIGINAL stream (pid+born+thread+from) - what duplicate compressions share */
+    originKey: string;
+};
+
+function parseLogFileName(name: string): ParsedName | undefined {
+    let match = /^pid(\d+)_born(\d+)_thread([\w-]+?)_from(\d+)(?:_to(\d+)_count(\d+)_id([a-z0-9]+))?\.jsonl(\.lz4)?$/.exec(name);
+    if (!match) return undefined;
+    let compressed = !!match[8];
+    // A compressed file without its compression fields (or the reverse) is not one of ours
+    if (compressed !== (match[5] !== undefined)) return undefined;
+    return {
+        pid: +match[1],
+        processStartTime: +match[2],
+        threadId: match[3],
+        startTime: +match[4],
+        endTime: match[5] !== undefined && +match[5] || undefined,
+        entryCount: match[6] !== undefined && +match[6] || undefined,
+        compressed,
+        originKey: `pid${match[1]}_born${match[2]}_thread${match[3]}_from${match[4]}`,
+    };
+}
+
+/** Whether the pid is alive AND is the same process incarnation the file name described. On Linux /proc gives the real start time; elsewhere a live pid is conservatively treated as the same process (never compress under a running process). */
+async function isProcessAlive(pid: number, processStartTime: number): Promise<boolean> {
+    try {
+        process.kill(pid, 0);
+    } catch {
+        return false;
+    }
+    try {
+        let stat = await fs.promises.stat(`/proc/${pid}`);
+        return Math.abs(stat.ctimeMs - processStartTime) < PROCESS_START_TOLERANCE;
+    } catch {
+        return true;
+    }
+}
+
+/**
+ * A searcher over one log file (as readFileCompressed returns it - always LZ4) that never decodes
+ * the whole thing: JSON escapes line breaks inside strings, so the only ACTUAL line breaks in the
+ * file are the separators between log statements - a plain substring search over the raw text,
+ * bounded out to the surrounding line breaks, finds exactly the matching statements, and only THOSE
+ * are decoded and returned as objects. Every search string must appear in a statement's raw JSON for
+ * it to match (so search for values as they are encoded - e.g. quoted).
+ */
+export function createLogSearcher(data: Buffer): (searches: string[]) => unknown[] {
+    let text = LZ4.decompress(data).toString("utf8");
+    return searches => {
+        if (!searches.length) {
+            throw new Error(`Provide at least one search string (an empty search would just be decodeLogFile)`);
+        }
+        let entries: unknown[] = [];
+        let index = 0;
+        while (true) {
+            let found = text.indexOf(searches[0], index);
+            if (found === -1) break;
+            let lineStart = text.lastIndexOf("\n", found) + 1;
+            let lineEnd = text.indexOf("\n", found);
+            if (lineEnd === -1) {
+                lineEnd = text.length;
+            }
+            // Past this line either way, so several matches inside one statement return it once
+            index = lineEnd + 1;
+            let line = text.slice(lineStart, lineEnd);
+            let matchesAll = true;
+            for (let other of searches.slice(1)) {
+                if (!line.includes(other)) {
+                    matchesAll = false;
+                    break;
+                }
+            }
+            if (!matchesAll) continue;
+            try {
+                entries.push(JSON.parse(line));
+            } catch {
+                // A torn final line (the writer crashed mid-append)
+                continue;
+            }
+        }
+        return entries;
+    };
+}
+
+/** Decodes a log file's bytes (as readFileCompressed returns them - always LZ4) back into the logged objects. A torn final line (the writer crashed mid-append) is skipped. */
+export function decodeLogFile(data: Buffer): unknown[] {
+    let text = LZ4.decompress(data).toString("utf8");
+    let entries: unknown[] = [];
+    for (let line of text.split("\n")) {
+        if (!line) continue;
+        try {
+            entries.push(JSON.parse(line));
+        } catch {
+            continue;
+        }
+    }
+    return entries;
+}
+
+export class StreamingLogs {
+    constructor(private config: {
+        folder: string;
+        /** Included in every file name - see misc/https/certs.ts getOwnThreadId. Processes without one write "none". */
+        threadId?: string;
+        maxFileBytes?: number;
+        totalLimitBytes?: number;
+    }) {
+        this.scheduleMaintenance();
+    }
+
+    private processStartTime = Math.round(Date.now() - process.uptime() * 1000);
+    private threadId = (this.config.threadId || "none").replace(/[^\w-]/g, "_");
+    private pending: string[] = [];
+    private flushTimer: ReturnType<typeof setTimeout> | undefined;
+    private maintenanceTimer: ReturnType<typeof setTimeout> | undefined;
+    // Serializes everything that touches the current file, so appends, rotation, and in-memory compression of the live file never interleave
+    private writeChain: Promise<void> = Promise.resolve();
+    private currentPath: string | undefined;
+    private currentBytes = 0;
+    private disposed = false;
+
+    /** Queues one entry (anything JSON-serializable). Never throws - logging must not take down the caller. */
+    public log(entry: unknown): void {
+        try {
+            if (this.pending.length >= MAX_PENDING_ENTRIES) {
+                // The disk is broken (flushes are failing) - dropping the oldest beats growing forever
+                this.pending.shift();
+            }
+            this.pending.push(JSON.stringify(entry));
+            this.scheduleFlush();
+        } catch (e) {
+            console.warn(`Could not queue a log entry (${(e as Error).message})`);
+        }
+    }
+
+    private scheduleFlush(): void {
+        if (this.flushTimer !== undefined || this.disposed) return;
+        this.flushTimer = setTimeout(() => {
+            this.flushTimer = undefined;
+            void this.flush().catch((e: Error) => console.warn(`Flushing logs in ${this.config.folder} failed: ${e.stack ?? e}`));
+        }, FLUSH_DELAY);
+        (this.flushTimer as { unref?: () => void }).unref?.();
+    }
+
+    public async flush(): Promise<void> {
+        let result = this.writeChain.then(() => this.writePending());
+        // The chain only serializes; a failure belongs to the caller that asked for this flush, not every one after it
+        this.writeChain = result.catch(() => { });
+        await result;
+    }
+
+    private async writePending(): Promise<void> {
+        if (!this.pending.length) return;
+        let lines = this.pending;
+        this.pending = [];
+        let data = lines.join("\n") + "\n";
+        try {
+            if (!this.currentPath) {
+                await fs.promises.mkdir(this.config.folder, { recursive: true });
+                this.currentPath = path.join(this.config.folder, `pid${process.pid}_born${this.processStartTime}_thread${this.threadId}_from${Date.now()}${UNCOMPRESSED_EXT}`);
+                this.currentBytes = 0;
+            }
+            await fs.promises.appendFile(this.currentPath, data);
+            this.currentBytes += Buffer.byteLength(data);
+        } catch (e) {
+            this.pending = lines.concat(this.pending);
+            this.scheduleFlush();
+            throw e;
+        }
+        if (this.currentBytes >= (this.config.maxFileBytes || MAX_FILE_BYTES)) {
+            let finished = this.currentPath;
+            this.currentPath = undefined;
+            this.currentBytes = 0;
+            void this.compressFile(finished).catch((e: Error) => {
+                // Left uncompressed: maintenance (ours after death, or any peer's) compresses it later
+                console.warn(`Compressing rotated log ${finished} failed: ${e.stack ?? e}`);
+            });
+        }
+    }
+
+    /**
+     * Compresses one finished log file: LZ4 into a temp SUBFOLDER (never a temp name in the log
+     * folder itself - a crashed write must not leave a corrupt file where the maintenance scans are),
+     * renamed into place (same drive, so the rename is atomic), verified, and only THEN is the
+     * original deleted - at no point is the data in fewer than one complete file.
+     */
+    private async compressFile(filePath: string): Promise<void> {
+        let name = path.basename(filePath);
+        let parsed = parseLogFileName(name);
+        if (!parsed || parsed.compressed) return;
+        let raw = await fs.promises.readFile(filePath);
+        let entryCount = 0;
+        for (let i = 0; i < raw.length; i++) {
+            if (raw[i] === 10) entryCount++;
+        }
+        let compressed = LZ4.compress(raw);
+        let id = Math.random().toString(36).slice(2, 10);
+        let finalName = `${parsed.originKey}_to${Date.now()}_count${entryCount}_id${id}${COMPRESSED_EXT}`;
+        let tempDir = path.join(this.config.folder, ".tmp");
+        await fs.promises.mkdir(tempDir, { recursive: true });
+        let tempPath = path.join(tempDir, finalName);
+        let finalPath = path.join(this.config.folder, finalName);
+        await fs.promises.writeFile(tempPath, compressed);
+        await fs.promises.rename(tempPath, finalPath);
+        // Paranoia before deleting the only other copy
+        await fs.promises.stat(finalPath);
+        await fs.promises.unlink(filePath);
+        console.log(`Compressed log ${name} -> ${finalName} (${raw.length} -> ${compressed.length} bytes, ${entryCount} entries)`);
+    }
+
+    // ── listing / reading ──
+
+    public async listFiles(): Promise<LogFileInfo[]> {
+        let names: string[];
+        try {
+            names = await fs.promises.readdir(this.config.folder);
+        } catch (e) {
+            if ((e as { code?: string }).code === "ENOENT") return [];
+            throw e;
+        }
+        let files: LogFileInfo[] = [];
+        for (let name of names) {
+            let parsed = parseLogFileName(name);
+            if (!parsed) continue;
+            let stat = await fs.promises.stat(path.join(this.config.folder, name)).catch(() => undefined);
+            if (!stat) continue;
+            let active = !parsed.compressed && await isProcessAlive(parsed.pid, parsed.processStartTime);
+            files.push({
+                name,
+                size: stat.size,
+                pid: parsed.pid,
+                processStartTime: parsed.processStartTime,
+                threadId: parsed.threadId,
+                startTime: parsed.startTime,
+                endTime: parsed.endTime,
+                entryCount: parsed.entryCount,
+                compressed: parsed.compressed,
+                active,
+            });
+        }
+        sort(files, x => x.startTime);
+        return files;
+    }
+
+    /** One file's bytes, ALWAYS LZ4-compressed: compressed files are sent as-is, a still-streaming file is flushed and compressed in memory (smaller over the network; decodeLogFile handles both identically). */
+    public async readFileCompressed(name: string): Promise<Buffer> {
+        let parsed = parseLogFileName(name);
+        if (!parsed || name.includes("/") || name.includes("\\")) {
+            throw new Error(`Not a log file name: ${JSON.stringify(name.slice(0, 200))}`);
+        }
+        let filePath = path.join(this.config.folder, name);
+        if (parsed.compressed) {
+            return await fs.promises.readFile(filePath);
+        }
+        // Flush so our own live file includes everything logged up to now
+        await this.flush();
+        return LZ4.compress(await fs.promises.readFile(filePath));
+    }
+
+    // ── maintenance ──
+
+    private scheduleMaintenance(): void {
+        if (this.disposed) return;
+        let wait = MAINTENANCE_MIN_INTERVAL + Math.random() * (MAINTENANCE_MAX_INTERVAL - MAINTENANCE_MIN_INTERVAL);
+        this.maintenanceTimer = setTimeout(() => {
+            void this.runMaintenance().catch((e: Error) => console.warn(`Log maintenance of ${this.config.folder} failed: ${e.stack ?? e}`)).finally(() => this.scheduleMaintenance());
+        }, wait);
+        (this.maintenanceTimer as { unref?: () => void }).unref?.();
+    }
+
+    /**
+     * Shared upkeep of the folder - every process using it runs this, so nothing depends on any one
+     * process surviving: dead processes' uncompressed files are compressed for them; duplicate
+     * compressions of one stream (two maintainers racing) are resolved by keeping the OLDEST copy;
+     * an uncompressed original whose compression already exists is deleted (its compressor died
+     * between rename and unlink); and the folder's total size is brought under its budget by
+     * deleting the oldest files.
+     */
+    public async runMaintenance(): Promise<void> {
+        let names: string[];
+        try {
+            names = await fs.promises.readdir(this.config.folder);
+        } catch (e) {
+            if ((e as { code?: string }).code === "ENOENT") return;
+            throw e;
+        }
+        let parsedByName = new Map<string, ParsedName>();
+        for (let name of names) {
+            let parsed = parseLogFileName(name);
+            if (parsed) parsedByName.set(name, parsed);
+        }
+        let compressedOrigins = new Map<string, string[]>();
+        for (let [name, parsed] of parsedByName) {
+            if (!parsed.compressed) continue;
+            let list = compressedOrigins.get(parsed.originKey);
+            if (!list) {
+                list = [];
+                compressedOrigins.set(parsed.originKey, list);
+            }
+            list.push(name);
+        }
+        // Duplicate compressions: keep the oldest (first written), delete the rest
+        for (let [origin, list] of compressedOrigins) {
+            if (list.length < 2) continue;
+            sort(list, name => parsedByName.get(name)?.endTime || 0);
+            for (let duplicate of list.slice(1)) {
+                console.log(`Deleting duplicate compressed log ${duplicate} (another maintainer compressed ${origin} first, keeping ${list[0]})`);
+                await fs.promises.unlink(path.join(this.config.folder, duplicate)).catch(() => { });
+                parsedByName.delete(duplicate);
+            }
+        }
+        for (let [name, parsed] of parsedByName) {
+            if (parsed.compressed) continue;
+            let filePath = path.join(this.config.folder, name);
+            if (filePath === this.currentPath) continue;
+            if (compressedOrigins.has(parsed.originKey)) {
+                // Already compressed by someone whose cleanup died between rename and unlink
+                console.log(`Deleting uncompressed log ${name}: its compressed version already exists`);
+                await fs.promises.unlink(filePath).catch(() => { });
+                continue;
+            }
+            // A live process owns its uncompressed file; only a DEAD process's files are compressed for it (our own pid always reads as alive, which correctly protects our current file - our rotated leftovers were excluded above only if current, so compress those too)
+            if (parsed.pid !== process.pid && await isProcessAlive(parsed.pid, parsed.processStartTime)) continue;
+            try {
+                await this.compressFile(filePath);
+            } catch (e) {
+                console.warn(`Compressing dead process's log ${name} failed (the next maintenance pass retries): ${(e as Error).stack ?? e}`);
+            }
+        }
+        await this.enforceTotalLimit();
+    }
+
+    private async enforceTotalLimit(): Promise<void> {
+        let limit = this.config.totalLimitBytes || TOTAL_LIMIT_BYTES;
+        let files = await this.listFiles();
+        let total = files.reduce((sum, x) => sum + x.size, 0);
+        if (total <= limit) return;
+        // Oldest first, by the end of what they cover; live files last (they are the newest data by definition)
+        let deletable = files.filter(x => !x.active);
+        sort(deletable, x => x.endTime || x.startTime);
+        let deleted = 0;
+        let deletedBytes = 0;
+        for (let file of deletable) {
+            if (total <= limit) break;
+            await fs.promises.unlink(path.join(this.config.folder, file.name)).catch(() => { });
+            total -= file.size;
+            deleted++;
+            deletedBytes += file.size;
+        }
+        console.log(`Logs in ${this.config.folder} outgrew their ${limit} byte budget: deleted the ${deleted} oldest files (${deletedBytes} bytes), ${total} bytes remain`);
+    }
+
+    public dispose(): void {
+        this.disposed = true;
+        if (this.flushTimer) {
+            clearTimeout(this.flushTimer);
+        }
+        if (this.maintenanceTimer) {
+            clearTimeout(this.maintenanceTimer);
+        }
+    }
+}

package/storage/TransactionFile.d.ts

@@ -4,10 +4,12 @@ export type LogEntry<T> = {
     time: number;
     changedAt: number;
 };
-/** A deleted key: when it was deleted, and when we learned. The value is gone; the time is the point of a tombstone. */
-export type LogTombstone = {
+/** A deleted key: when it was deleted, and when we learned. When the key had a live value at deletion time it is MARKED rather than gone - the value (and its original write time) rides along, so the underlying data can still be read and the deletion can be undone (see unmark) until the history is dropped (see dropValue). */
+export type LogTombstone<T> = {
     time: number;
     changedAt: number;
+    value?: T;
+    valueTime?: number;
 };
 export declare class TransactionFile<T> {
     private filePath;
@@ -26,8 +28,8 @@ export declare class TransactionFile<T> {
     };
     /** The live value, or undefined when the key does not exist here (deleted included - a deletion is an absence, see getDeleted for its time). */
     get(key: string): LogEntry<T> | undefined;
-    /** When the key was deleted, if it was. Absent both here and in get means we have never heard of it. */
-    getDeleted(key: string): LogTombstone | undefined;
+    /** When the key was deleted, if it was (value included when the deletion is still marked - see LogTombstone). Absent both here and in get means we have never heard of it. */
+    getDeleted(key: string): LogTombstone<T> | undefined;
     /** The time the key last changed either way, or 0 if we have never heard of it - what a new write has to beat. */
     timeOf(key: string): number;
     /** O(1), and counts only what exists. */
@@ -36,11 +38,15 @@ export declare class TransactionFile<T> {
     /** Live values only. Live, in insertion order - deleting during iteration is safe (JS skips entries removed before they are reached), which is what the passes that walk everything and prune as they go rely on. */
     entries(): IterableIterator<[string, LogEntry<T>]>;
     /** The tombstones, which is a much smaller walk than the values - so expiring them, or listing what was deleted since some time, costs what it should. */
-    deletedEntries(): IterableIterator<[string, LogTombstone]>;
+    deletedEntries(): IterableIterator<[string, LogTombstone<T>]>;
     /** Stores a value as of `time`. Returns false when something at least as new is already here, in which case nothing changed - an out-of-order write is not an error, it is just late. */
     set(key: string, value: T, time: number): boolean;
-    /** Deletes as of `time`, keeping the tombstone. Returns false when something at least as new is already here. */
+    /** Deletes as of `time`, keeping the tombstone. A key that had a live value keeps it in the tombstone as MARKED for deletion (readable and restorable until dropValue). Returns false when something at least as new is already here. */
     delete(key: string, time: number): boolean;
+    /** Undoes a marked deletion: the kept value becomes live again, as of `time` (a fresh time, so the restore outranks the deletion everywhere it propagated). Returns false when there is no marked value to restore, or something at least as new is already here. */
+    unmark(key: string, time: number): boolean;
+    /** Drops a marked deletion's kept value (its history has been physically removed), leaving a plain tombstone with the same delete time. */
+    dropValue(key: string): void;
     /** Forgets the key entirely, tombstone included - for a tombstone old enough that nobody needs to hear about the deletion any more, and for an entry that turned out never to have existed. Not a deletion: it leaves nothing behind to propagate. */
     purge(key: string): void;
     private applySet;

package/storage/TransactionFile.ts

@@ -15,8 +15,8 @@ const FLUSH_DELAY = 500;
 
 /** A live value: what was stored, when it was written (the caller's ordering), and when we last changed it (ours). */
 export type LogEntry<T> = { value: T; time: number; changedAt: number };
-/** A deleted key: when it was deleted, and when we learned. The value is gone; the time is the point of a tombstone. */
-export type LogTombstone = { time: number; changedAt: number };
+/** A deleted key: when it was deleted, and when we learned. When the key had a live value at deletion time it is MARKED rather than gone - the value (and its original write time) rides along, so the underlying data can still be read and the deletion can be undone (see unmark) until the history is dropped (see dropValue). */
+export type LogTombstone<T> = { time: number; changedAt: number; value?: T; valueTime?: number };
 
 // One line of the log, with single-letter field names because there is one record per set/delete/purge ever made and the field names are most of a record's overhead. JSON.stringify escapes newlines inside keys and values, so a record can never contain the separator.
 type LogRecord<T> = {
@@ -28,6 +28,10 @@ type LogRecord<T> = {
     v?: T;
     // deleted, the file on disk is deleted, preserved when we compact
     d?: 1;
+    // marked for deletion: deleted, but the value (v) and its original write time (w) are kept, so the data is still readable and restorable. Preserved when we compact.
+    m?: 1;
+    // writeTime, the original write time of a marked-for-deletion value (t is the DELETE time on those records)
+    w?: number;
     // purged, the file on disk is deleted, not preserved when we compact
     p?: 1;
 };
@@ -36,7 +40,7 @@ export class TransactionFile<T> {
     constructor(private filePath: string) { }
 
     private values = new Map<string, LogEntry<T>>();
-    private deleted = new Map<string, LogTombstone>();
+    private deleted = new Map<string, LogTombstone<T>>();
     // Records the log holds (written plus pending) - what compaction is decided against, NOT the number of keys
     private logRecords = 0;
     private pending: string[] = [];
@@ -70,6 +74,8 @@ export class TransactionFile<T> {
             if (record.p) {
                 this.values.delete(record.k);
                 this.deleted.delete(record.k);
+            } else if (record.m) {
+                this.applyDelete(record.k, record.t, record.t, record.v, record.w);
             } else if (record.d) {
                 this.applyDelete(record.k, record.t, record.t);
             } else if (record.v !== undefined) {
@@ -86,8 +92,8 @@ export class TransactionFile<T> {
     public get(key: string): LogEntry<T> | undefined {
         return this.values.get(key);
     }
-    /** When the key was deleted, if it was. Absent both here and in get means we have never heard of it. */
-    public getDeleted(key: string): LogTombstone | undefined {
+    /** When the key was deleted, if it was (value included when the deletion is still marked - see LogTombstone). Absent both here and in get means we have never heard of it. */
+    public getDeleted(key: string): LogTombstone<T> | undefined {
         return this.deleted.get(key);
     }
     /** The time the key last changed either way, or 0 if we have never heard of it - what a new write has to beat. */
@@ -109,7 +115,7 @@ export class TransactionFile<T> {
         return this.values.entries();
     }
     /** The tombstones, which is a much smaller walk than the values - so expiring them, or listing what was deleted since some time, costs what it should. */
-    public deletedEntries(): IterableIterator<[string, LogTombstone]> {
+    public deletedEntries(): IterableIterator<[string, LogTombstone<T>]> {
         return this.deleted.entries();
     }
 
@@ -120,13 +126,35 @@ export class TransactionFile<T> {
         return true;
     }
 
-    /** Deletes as of `time`, keeping the tombstone. Returns false when something at least as new is already here. */
+    /** Deletes as of `time`, keeping the tombstone. A key that had a live value keeps it in the tombstone as MARKED for deletion (readable and restorable until dropValue). Returns false when something at least as new is already here. */
     public delete(key: string, time: number): boolean {
-        if (!this.applyDelete(key, time, Date.now())) return false;
-        this.append({ k: key, t: time, d: 1 });
+        let live = this.values.get(key);
+        if (!this.applyDelete(key, time, Date.now(), live?.value, live?.time)) return false;
+        if (live) {
+            this.append({ k: key, t: time, m: 1, v: live.value, w: live.time });
+        } else {
+            this.append({ k: key, t: time, d: 1 });
+        }
+        return true;
+    }
+
+    /** Undoes a marked deletion: the kept value becomes live again, as of `time` (a fresh time, so the restore outranks the deletion everywhere it propagated). Returns false when there is no marked value to restore, or something at least as new is already here. */
+    public unmark(key: string, time: number): boolean {
+        let tombstone = this.deleted.get(key);
+        if (!tombstone || tombstone.value === undefined) return false;
+        if (!this.applySet(key, tombstone.value, time, Date.now())) return false;
+        this.append({ k: key, t: time, v: tombstone.value });
         return true;
     }
 
+    /** Drops a marked deletion's kept value (its history has been physically removed), leaving a plain tombstone with the same delete time. */
+    public dropValue(key: string): void {
+        let tombstone = this.deleted.get(key);
+        if (!tombstone || tombstone.value === undefined) return;
+        this.deleted.set(key, { time: tombstone.time, changedAt: Date.now() });
+        this.append({ k: key, t: tombstone.time, d: 1 });
+    }
+
     /** Forgets the key entirely, tombstone included - for a tombstone old enough that nobody needs to hear about the deletion any more, and for an entry that turned out never to have existed. Not a deletion: it leaves nothing behind to propagate. */
     public purge(key: string): void {
         let had = this.values.delete(key);
@@ -142,10 +170,10 @@ export class TransactionFile<T> {
         return true;
     }
 
-    private applyDelete(key: string, time: number, changedAt: number): boolean {
+    private applyDelete(key: string, time: number, changedAt: number, value?: T, valueTime?: number): boolean {
         if (time < this.timeOf(key)) return false;
         this.values.delete(key);
-        this.deleted.set(key, { time, changedAt });
+        this.deleted.set(key, { time, changedAt, value, valueTime });
         return true;
     }
 
@@ -206,7 +234,11 @@ export class TransactionFile<T> {
             records.push(JSON.stringify({ k: key, t: entry.time, v: entry.value } satisfies LogRecord<T>));
         }
         for (let [key, tombstone] of this.deleted) {
-            records.push(JSON.stringify({ k: key, t: tombstone.time, d: 1 } satisfies LogRecord<T>));
+            if (tombstone.value !== undefined) {
+                records.push(JSON.stringify({ k: key, t: tombstone.time, m: 1, v: tombstone.value, w: tombstone.valueTime } satisfies LogRecord<T>));
+            } else {
+                records.push(JSON.stringify({ k: key, t: tombstone.time, d: 1 } satisfies LogRecord<T>));
+            }
         }
         // Cleared with the snapshot: every pending record is already reflected in the maps, so the snapshot contains it. Anything appended from here on is NOT in the snapshot, and lands in the new log after the rename.
         this.pending = [];