sliftutils 1.4.69 → 1.4.71

package/index.d.ts

@@ -940,7 +940,6 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         streamFoldHardLimitBytes: number;
         writeFlushMaxDelayMs: number;
         fileSetPollIntervalMs: number;
-        deleteDeferMs: number;
         memoryFlushHeapBytes: number;
         memoryFlushMinCollectionBytes: number;
         memoryFlushThrottleMs: number;
@@ -1023,6 +1022,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
             key: string;
         })[]): Promise<void>;
         private listFiles;
+        private processMarkers;
         private writeBulkFile;
         private maybeMerge;
         tryMergeNow: () => Promise<void>;
@@ -1532,6 +1532,25 @@ declare module "sliftutils/storage/BulkDatabase2/mergeLock" {
 
 }
 
+declare module "sliftutils/storage/BulkDatabase2/mergeMarkers" {
+    import type { FileStorage } from "../FileFolderAPI";
+    export type DeleteMarker = {
+        fileName: string;
+        deleteFiles: string[];
+        replacedBy: string[];
+        time: number;
+    };
+    export declare function isMarkerFile(name: string): boolean;
+    export declare function writeDeleteMarker(storage: FileStorage, config: {
+        deleteFiles: string[];
+        replacedBy: string[];
+    }): Promise<void>;
+    export declare function readDeleteMarkers(storage: FileStorage, allNames: string[]): Promise<DeleteMarker[]>;
+    export declare function markerExclusions(markers: DeleteMarker[]): Set<string>;
+    export declare function processDeleteMarkers(storage: FileStorage, markers: DeleteMarker[], allNames: string[]): Promise<void>;
+
+}
+
 declare module "sliftutils/storage/BulkDatabase2/streamLog" {
     /// <reference types="node" />
     /// <reference types="node" />

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.69",
+    "version": "1.4.71",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -12,7 +12,6 @@ export declare const bulkDatabase2Timing: {
     streamFoldHardLimitBytes: number;
     writeFlushMaxDelayMs: number;
     fileSetPollIntervalMs: number;
-    deleteDeferMs: number;
     memoryFlushHeapBytes: number;
     memoryFlushMinCollectionBytes: number;
     memoryFlushThrottleMs: number;
@@ -95,6 +94,7 @@ export declare class BulkDatabaseBase<T extends {
         key: string;
     })[]): Promise<void>;
     private listFiles;
+    private processMarkers;
     private writeBulkFile;
     private maybeMerge;
     tryMergeNow: () => Promise<void>;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -19,6 +19,7 @@ import { STREAM_EXTENSION, frameDeletes, frameRows, streamReaderFromEntries } fr
 import { broadcast as syncBroadcast, broadcastSeal as syncBroadcastSeal, connect as syncConnect, isSyncSupported, RemoteWrite } from "./syncClient";
 import { DELETED } from "./WriteOverlay";
 import { releaseMergeFileLock, releaseMergeLock, startMergeFileLockHeartbeat, tryAcquireMergeFileLock, tryAcquireMergeLock } from "./mergeLock";
+import { markerExclusions, processDeleteMarkers, readDeleteMarkers, writeDeleteMarker } from "./mergeMarkers";
 import {
     BulkFileInfo,
     LoadedIndex,
@@ -42,11 +43,11 @@ const MAX_INDEX_RELOAD_ATTEMPTS = 3;
 const FIRST_MERGE_BYTES = TARGET_FILE_BYTES / 2;
 const KEY_GROUP_BYTES = 800 * 1024 * 1024;
 const DUP_THRESHOLD = 0.4;
-// Emergency Pass 2 short-circuit: once Pass 1 has had a chance to fold streams, if the bulk tier is
-// big enough AND the overall key duplication fraction is high, do ONE merge across every bulk file
-// instead of the per-key-group walk (which spaces merges 5 min apart — 16 hours for 200 groups).
-const EMERGENCY_DEDUP_BYTES = 512 * 1024 * 1024;
-const EMERGENCY_DEDUP_FRACTION = 0.5;
+// Whole-tier dedup short-circuit (after Pass 1): when the bulk tier is over this size AND the overall
+// key duplication fraction is over this threshold, fold every bulk file in one merge instead of the
+// per-key-group walk (which spaces merges 5 min apart — 16 h for 200 groups).
+const DEDUP_TRIGGER_BYTES = 512 * 1024 * 1024;
+const DEDUP_TRIGGER_FRACTION = 0.5;
 const WRITE_FLUSH_FIRST_STEP_MS = 250;
 
 export const bulkDatabase2Timing = {
@@ -66,12 +67,6 @@ export const bulkDatabase2Timing = {
     // the whole stream file per write.
     writeFlushMaxDelayMs: isNode() ? 0 : 15 * 1000,
     fileSetPollIntervalMs: 30 * 60 * 1000,
-    // Defer deletion of a merge's consumed inputs by this long. The OS may not durably flush the new
-    // outputs immediately, so if we deleted the inputs and crashed before the writes settled we'd lose
-    // data. Within the window the inputs sit as duplicates of the outputs; if we never crash, the timer
-    // fires and removes them. If we DO crash before the timer, the duplicates stay until the next
-    // key-stratify pass (high duplicate-key fraction → it merges them again, dedup completes).
-    deleteDeferMs: 15 * 60 * 1000,
     memoryFlushHeapBytes: 1500 * 1024 * 1024,
     memoryFlushMinCollectionBytes: 100 * 1024 * 1024,
     memoryFlushThrottleMs: 15 * 60 * 1000,
@@ -550,9 +545,16 @@ export class BulkDatabaseBase<T extends { key: string }> {
     private async listFiles(): Promise<{ bulkFiles: BulkFileInfo[]; streamFiles: StreamFileInfo[] }> {
         const storage = await this.storage();
         const names = await storage.getKeys();
+        // A consumed-merge input is hidden from reads the instant its deletion marker exists — that's
+        // what stops it from being re-read and re-merged. Physical deletion + marker cleanup runs on a
+        // throttle (not inline) so a read isn't slowed by it.
+        const markers = await readDeleteMarkers(storage, names);
+        const excluded = markerExclusions(markers);
+        if (markers.length) void this.processMarkers();
         const bulkFiles: BulkFileInfo[] = [];
         const streamFiles: StreamFileInfo[] = [];
         for (const n of names) {
+            if (excluded.has(n)) continue;
             if (n.endsWith(FILE_EXTENSION)) { const p = parseFileName(n); if (p) bulkFiles.push(p); }
             else if (n.endsWith(STREAM_EXTENSION)) { const p = parseStreamFileName(n); if (p) streamFiles.push(p); }
         }
@@ -564,6 +566,16 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return { bulkFiles, streamFiles };
     }
 
+    // Throttled marker housekeeping: delete inputs whose replacement outputs have landed (or whose
+    // marker has aged out) and retire fulfilled markers. Triggered from listFiles, so it's gated to
+    // avoid a delete storm under heavy reads.
+    private processMarkers = throttleFunction(30 * 1000, async () => {
+        const storage = await this.storage();
+        const names = await storage.getKeys();
+        const markers = await readDeleteMarkers(storage, names);
+        if (markers.length) await processDeleteMarkers(storage, markers, names);
+    });
+
     private async writeBulkFile(rows: Record<string, unknown>[]): Promise<void> {
         const storage = await this.storage();
         const timestamp = nextFileTime();
@@ -809,36 +821,29 @@ export class BulkDatabaseBase<T extends { key: string }> {
         for (const line of steps) console.log(line);
         console.groupEnd();
 
-        // Rebuild + swap NOW so the index sees the new outputs; the old inputs are still on disk until
-        // the deferred-delete timer fires (the index briefly serves both, resolved by time so reads are
-        // correct). Block-cache eviction for inputs happens on the SECOND rebuild after deletion.
-        await this.triggerRebuild();
-
-        // Only the sources runPlannedMerge actually used are safe to delete — a source it dropped
-        // mid-plan (file gone, corruption) still holds data we couldn't merge in, so leave it on disk
-        // and let a future merge re-attempt it.
+        // Only the sources runPlannedMerge actually used can be retired — a source it dropped mid-plan
+        // (file gone, corruption) still holds data we couldn't merge in, so leave it on disk and let a
+        // future merge re-attempt it.
         const usedConsumedBulk = consumedBulk.filter(f => mergeResult.usedSourceNames.has(f.fileName));
         const usedStreamFiles = streamFiles.filter(f => mergeResult.usedSourceNames.has(f.fileName) || mergeResult.usedSourceNames.has("(streams)"));
+        // A stream may still be appended to after we read it (the writer hasn't sealed/moved on); only
+        // retire streams canDeleteStream clears — the rest stay live and a later merge re-folds them
+        // once sealed. Bulk files are immutable, so a used one is always safe to retire.
+        const deletableStreams: string[] = [];
+        for (const f of usedStreamFiles) {
+            if (await this.canDeleteStream(f, Date.now(), streamData.sizes, forceDeleteStreams)) deletableStreams.push(f.fileName);
+        }
+        const deleteFiles = [...usedConsumedBulk.map(f => f.fileName), ...deletableStreams];
 
-        // Deferred delete: give the FS time to durably flush new writes before removing the inputs.
-        // If we crash before the timer fires, the inputs survive as duplicates and a later key-stratify
-        // pass dedupes them. We DON'T await this — the merge returns "done" as soon as the new outputs
-        // are written and the index is swapped in.
-        const remove = async (name: string) => { try { await storage.remove(name); } catch { /* already gone */ } };
-        const timer = setTimeout(() => {
-            void (async () => {
-                try {
-                    for (const f of usedConsumedBulk) await remove(f.fileName);
-                    for (const f of usedStreamFiles) {
-                        if (await this.canDeleteStream(f, Date.now(), streamData.sizes, forceDeleteStreams)) await remove(f.fileName);
-                    }
-                    await this.triggerRebuild();
-                } catch (e) {
-                    console.warn(`${this.name}: deferred delete of consumed merge inputs failed: ${(e as Error).message}`);
-                }
-            })();
-        }, bulkDatabase2Timing.deleteDeferMs);
-        (timer as { unref?: () => void }).unref?.();
+        // Write a deletion marker instead of deleting inline. From the next read on, listFiles hides
+        // these inputs (so they're never re-merged — the loop fix), and processMarkers removes them
+        // physically once the outputs have landed or the marker ages out. Marker BEFORE the rebuild so
+        // the freshly-swapped index already excludes the retired inputs.
+        if (deleteFiles.length) await writeDeleteMarker(storage, { deleteFiles, replacedBy: outNames });
+
+        // Rebuild + swap so the index sees the new outputs and drops the retired inputs. Block-cache
+        // eviction for the dropped inputs happens here (they're no longer in the new index's fileSet).
+        await this.triggerRebuild();
         return newNames.length > 0 || carriedDeletes > 0;
     }
 
@@ -941,9 +946,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
             }
         }
 
-        // Pass 1.5 (emergency dedup): Pass 1 has had its chance. If the bulk tier is fat AND mostly
-        // duplicates, the per-group walk below (5 min spacing per group) takes hours for an extreme
-        // state (1000+ files, 99% dup) — fold every bulk file in one merge and skip the walk.
+        // Whole-tier dedup short-circuit (between Pass 1 and the per-group Pass 2): when the bulk tier
+        // is big enough AND mostly duplicates, fold every bulk file in one merge. Avoids the per-group
+        // walk's 5-min spacing × N-groups latency.
         {
             const { bulkFiles } = await this.listFiles();
             if (bulkFiles.length >= 2) {
@@ -960,8 +965,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
                     } catch { /* skip unreadable */ }
                 }
                 const dupFraction = totalSlots ? (totalSlots - uniqueKeys.size) / totalSlots : 0;
-                if (totalBytes > EMERGENCY_DEDUP_BYTES && dupFraction > EMERGENCY_DEDUP_FRACTION) {
-                    console.log(`${blue(this.name)} ${magenta("emergency-dedup")}: ${fmtBytes(totalBytes)} across ${bulkFiles.length} bulk file(s), ${Math.round(dupFraction * 100)}% duplicate key-slots — folding all at once`);
+                if (totalBytes > DEDUP_TRIGGER_BYTES && dupFraction > DEDUP_TRIGGER_FRACTION) {
+                    console.log(`${blue(this.name)} ${magenta("dedup")}: ${fmtBytes(totalBytes)} across ${bulkFiles.length} bulk file(s), ${Math.round(dupFraction * 100)}% duplicate key-slots — folding all at once`);
                     if (!await runMerge(bulkFiles, [])) return merged;
                     return merged;
                 }

package/storage/BulkDatabase2/mergeMarkers.d.ts

@@ -0,0 +1,15 @@
+import type { FileStorage } from "../FileFolderAPI";
+export type DeleteMarker = {
+    fileName: string;
+    deleteFiles: string[];
+    replacedBy: string[];
+    time: number;
+};
+export declare function isMarkerFile(name: string): boolean;
+export declare function writeDeleteMarker(storage: FileStorage, config: {
+    deleteFiles: string[];
+    replacedBy: string[];
+}): Promise<void>;
+export declare function readDeleteMarkers(storage: FileStorage, allNames: string[]): Promise<DeleteMarker[]>;
+export declare function markerExclusions(markers: DeleteMarker[]): Set<string>;
+export declare function processDeleteMarkers(storage: FileStorage, markers: DeleteMarker[], allNames: string[]): Promise<void>;

package/storage/BulkDatabase2/mergeMarkers.ts

@@ -0,0 +1,89 @@
+// Deletion markers: how a merge retires the input files it consumed WITHOUT deleting them inline.
+//
+// The old approach deleted consumed inputs on a 15-min timer. If the tab closed before the timer fired
+// the inputs survived AND were still read — so the next merge re-merged the same duplicates, forever
+// (the "1739 files, 155GB, 99% duplicate" loop). Markers fix both halves: the consumed inputs are
+// EXCLUDED from reads the moment the marker exists (so they're never re-merged), and their physical
+// deletion is driven off the marker on a later read instead of an in-process timer.
+//
+// A marker names the input files it wants gone (`deleteFiles`) and the output files that supersede them
+// (`replacedBy`). On every read we:
+//   1) exclude every `deleteFiles` entry from the active file set (unconditionally — that's the loop fix);
+//   2) delete a marker's `deleteFiles` once its `replacedBy` outputs all exist OR the marker is older than
+//      the grace window (a crash-safety floor so a marker whose outputs never landed still gets cleaned);
+//   3) once a marker's `deleteFiles` are all gone, and the marker has aged past the grace window, delete
+//      the marker itself (kept around briefly so a concurrent reader still excludes the files consistently).
+
+import type { FileStorage } from "../FileFolderAPI";
+
+const MARKER_PREFIX = ".delete-marker_";
+const MARKER_GRACE_MS = 5 * 60 * 1000;
+
+export type DeleteMarker = {
+    fileName: string;
+    deleteFiles: string[];
+    replacedBy: string[];
+    time: number;
+};
+
+export function isMarkerFile(name: string): boolean {
+    return name.startsWith(MARKER_PREFIX);
+}
+
+export async function writeDeleteMarker(storage: FileStorage, config: { deleteFiles: string[]; replacedBy: string[] }): Promise<void> {
+    const time = Date.now();
+    const name = `${MARKER_PREFIX}${time}_${Math.random().toString(36).slice(2, 10)}`;
+    const body = JSON.stringify({ deleteFiles: config.deleteFiles, replacedBy: config.replacedBy, time });
+    await storage.set(name, Buffer.from(body, "utf8") as Buffer);
+}
+
+export async function readDeleteMarkers(storage: FileStorage, allNames: string[]): Promise<DeleteMarker[]> {
+    const markerNames = allNames.filter(isMarkerFile);
+    const markers = await Promise.all(markerNames.map(async (fileName): Promise<DeleteMarker | undefined> => {
+        try {
+            const buf = await storage.get(fileName);
+            if (!buf) return undefined;
+            const parsed = JSON.parse(buf.toString("utf8")) as { deleteFiles?: unknown; replacedBy?: unknown; time?: unknown };
+            if (!Array.isArray(parsed.deleteFiles) || !Array.isArray(parsed.replacedBy)) return undefined;
+            return {
+                fileName,
+                deleteFiles: parsed.deleteFiles as string[],
+                replacedBy: parsed.replacedBy as string[],
+                time: typeof parsed.time === "number" ? parsed.time : 0,
+            };
+        } catch {
+            return undefined;
+        }
+    }));
+    return markers.filter((m): m is DeleteMarker => !!m);
+}
+
+// The set of files that any marker wants deleted — these must be dropped from the active read set.
+export function markerExclusions(markers: DeleteMarker[]): Set<string> {
+    const excluded = new Set<string>();
+    for (const m of markers) for (const n of m.deleteFiles) excluded.add(n);
+    return excluded;
+}
+
+// Act on each marker (see file header). Idempotent and best-effort: every removal is guarded, so two
+// processes (or two reads) running this at once just race to the same already-correct end state.
+export async function processDeleteMarkers(storage: FileStorage, markers: DeleteMarker[], allNames: string[]): Promise<void> {
+    const present = new Set(allNames);
+    const now = Date.now();
+    for (const marker of markers) {
+        const stillPresent = marker.deleteFiles.filter(n => present.has(n));
+        if (!stillPresent.length) {
+            // The files this marker retired are gone; once it's aged past the grace window, retire the marker too.
+            if (now - marker.time > MARKER_GRACE_MS) {
+                try { await storage.remove(marker.fileName); } catch { /* already gone */ }
+            }
+            continue;
+        }
+        const replacedExists = marker.replacedBy.length > 0 && marker.replacedBy.every(n => present.has(n));
+        if (replacedExists || now - marker.time > MARKER_GRACE_MS) {
+            for (const n of stillPresent) {
+                try { await storage.remove(n); } catch { /* already gone */ }
+            }
+        }
+    }
+}