sliftutils 1.4.65 → 1.4.67

package/index.d.ts

@@ -853,6 +853,13 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
         isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
         /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
         isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
+        /**
+         * Reactive: true while a merge is rewriting this collection's files (background `maybeMerge` or
+         * an explicit `compact`/`merge`/`tryMergeNow`). Becomes false as soon as the new index is swapped
+         * in — the deferred-delete cleanup window is NOT counted. Use this in a UI to show a per-database
+         * "compacting…" indicator.
+         */
+        isCompactingSync(): boolean;
         /**
          * Whether a row (key) is currently being watched by some reactive observer (getSingleFieldObjSync /
          * getSingleFieldSync). Lets callers skip per-row work when nothing's watching. Non-reactive query;
@@ -936,6 +943,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         streamFoldHardLimitBytes: number;
         writeFlushMaxDelayMs: number;
         fileSetPollIntervalMs: number;
+        deleteDeferMs: number;
         memoryFlushHeapBytes: number;
         memoryFlushMinCollectionBytes: number;
         memoryFlushThrottleMs: number;
@@ -1029,6 +1037,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private fileLogicalSize;
         private handleUnreadableFile;
         private mergeFileSet;
+        private mergeFileSetInner;
         private canDeleteStream;
         private mergeSpacingDelay;
         private testMerge;
@@ -1058,6 +1067,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         }[] | undefined;
         isFieldLoadedSync<C extends keyof T>(key: string, column: C): boolean;
         isColumnLoadedSync<C extends keyof T>(column: C): boolean;
+        isCompactingSync(): boolean;
         getColumnInfo(): Promise<{
             column: string;
             byteSize: number;
@@ -1278,6 +1288,10 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseReader" {
         isKeyWatched(key: string): boolean;
         isLiveNow(key: string): boolean;
         localTime(key: string): number;
+        private compactingCount;
+        beginCompaction(): void;
+        endCompaction(): void;
+        isCompactingSync(): boolean;
         private notifyOverlayMutation;
         getKeys(): Promise<string[]>;
         getColumn<C extends keyof T>(column: C): Promise<{

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.65",
+    "version": "1.4.67",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabase2.d.ts

@@ -88,6 +88,13 @@ export interface IBulkDatabase2<T extends {
     isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
     /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
     isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
+    /**
+     * Reactive: true while a merge is rewriting this collection's files (background `maybeMerge` or
+     * an explicit `compact`/`merge`/`tryMergeNow`). Becomes false as soon as the new index is swapped
+     * in — the deferred-delete cleanup window is NOT counted. Use this in a UI to show a per-database
+     * "compacting…" indicator.
+     */
+    isCompactingSync(): boolean;
     /**
      * Whether a row (key) is currently being watched by some reactive observer (getSingleFieldObjSync /
      * getSingleFieldSync). Lets callers skip per-row work when nothing's watching. Non-reactive query;

package/storage/BulkDatabase2/BulkDatabase2.ts

@@ -74,6 +74,14 @@ export interface IBulkDatabase2<T extends { key: string }> {
     /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
     isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
 
+    /**
+     * Reactive: true while a merge is rewriting this collection's files (background `maybeMerge` or
+     * an explicit `compact`/`merge`/`tryMergeNow`). Becomes false as soon as the new index is swapped
+     * in — the deferred-delete cleanup window is NOT counted. Use this in a UI to show a per-database
+     * "compacting…" indicator.
+     */
+    isCompactingSync(): boolean;
+
     /**
      * Whether a row (key) is currently being watched by some reactive observer (getSingleFieldObjSync /
      * getSingleFieldSync). Lets callers skip per-row work when nothing's watching. Non-reactive query;

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -12,6 +12,7 @@ export declare const bulkDatabase2Timing: {
     streamFoldHardLimitBytes: number;
     writeFlushMaxDelayMs: number;
     fileSetPollIntervalMs: number;
+    deleteDeferMs: number;
     memoryFlushHeapBytes: number;
     memoryFlushMinCollectionBytes: number;
     memoryFlushThrottleMs: number;
@@ -105,6 +106,7 @@ export declare class BulkDatabaseBase<T extends {
     private fileLogicalSize;
     private handleUnreadableFile;
     private mergeFileSet;
+    private mergeFileSetInner;
     private canDeleteStream;
     private mergeSpacingDelay;
     private testMerge;
@@ -134,6 +136,7 @@ export declare class BulkDatabaseBase<T extends {
     }[] | undefined;
     isFieldLoadedSync<C extends keyof T>(key: string, column: C): boolean;
     isColumnLoadedSync<C extends keyof T>(column: C): boolean;
+    isCompactingSync(): boolean;
     getColumnInfo(): Promise<{
         column: string;
         byteSize: number;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -57,6 +57,12 @@ export const bulkDatabase2Timing = {
     // the whole stream file per write.
     writeFlushMaxDelayMs: isNode() ? 0 : 15 * 1000,
     fileSetPollIntervalMs: 30 * 60 * 1000,
+    // Defer deletion of a merge's consumed inputs by this long. The OS may not durably flush the new
+    // outputs immediately, so if we deleted the inputs and crashed before the writes settled we'd lose
+    // data. Within the window the inputs sit as duplicates of the outputs; if we never crash, the timer
+    // fires and removes them. If we DO crash before the timer, the duplicates stay until the next
+    // key-stratify pass (high duplicate-key fraction → it merges them again, dedup completes).
+    deleteDeferMs: 15 * 60 * 1000,
     memoryFlushHeapBytes: 1500 * 1024 * 1024,
     memoryFlushMinCollectionBytes: 100 * 1024 * 1024,
     memoryFlushThrottleMs: 15 * 60 * 1000,
@@ -626,9 +632,17 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // we trigger an index rebuild + atomic swap; once swap completes, the consumed files' block-cache
     // entries are evicted (no consumer can ask for them now).
     private async mergeFileSet(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest = false, forceDeleteStreams = false): Promise<boolean> {
+        this.reader.beginCompaction();
+        try {
+            return await this.mergeFileSetInner(bulkFiles, streamFiles, includesOldest, forceDeleteStreams);
+        } finally {
+            this.reader.endCompaction();
+        }
+    }
+
+    private async mergeFileSetInner(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest: boolean, forceDeleteStreams: boolean): Promise<boolean> {
         const storage = await this.storage();
         const timestamp = nextFileTime();
-        const now = Date.now();
 
         const consumedBulk: BulkFileInfo[] = [];
         const bulkReaders: BaseBulkDatabaseReader[] = [];
@@ -702,15 +716,30 @@ export class BulkDatabaseBase<T extends { key: string }> {
         for (const line of steps) console.log(line);
         console.groupEnd();
 
-        const remove = async (name: string) => { try { await storage.remove(name); } catch { /* already gone */ } };
-        for (const f of consumedBulk) await remove(f.fileName);
-        for (const f of streamFiles) {
-            if (await this.canDeleteStream(f, now, streamData.sizes, forceDeleteStreams)) await remove(f.fileName);
-        }
-
-        // File set changed — rebuild + swap. After the swap, consumed files' block-cache entries are
-        // evicted (no reader will request them now).
+        // Rebuild + swap NOW so the index sees the new outputs; the old inputs are still on disk until
+        // the deferred-delete timer fires (the index briefly serves both, resolved by time so reads are
+        // correct). Block-cache eviction for inputs happens on the SECOND rebuild after deletion.
         await this.triggerRebuild();
+
+        // Deferred delete: give the FS time to durably flush new writes before removing the inputs.
+        // If we crash before the timer fires, the inputs survive as duplicates and a later key-stratify
+        // pass dedupes them. We DON'T await this — the merge returns "done" as soon as the new outputs
+        // are written and the index is swapped in.
+        const remove = async (name: string) => { try { await storage.remove(name); } catch { /* already gone */ } };
+        const timer = setTimeout(() => {
+            void (async () => {
+                try {
+                    for (const f of consumedBulk) await remove(f.fileName);
+                    for (const f of streamFiles) {
+                        if (await this.canDeleteStream(f, Date.now(), streamData.sizes, forceDeleteStreams)) await remove(f.fileName);
+                    }
+                    await this.triggerRebuild();
+                } catch (e) {
+                    console.warn(`${this.name}: deferred delete of consumed merge inputs failed: ${(e as Error).message}`);
+                }
+            })();
+        }, bulkDatabase2Timing.deleteDeferMs);
+        (timer as { unref?: () => void }).unref?.();
         return newNames.length > 0 || carriedDeletes > 0;
     }
 
@@ -912,6 +941,13 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return this.reader.isColumnLoadedSync(column);
     }
 
+    // Reactive: true while a merge is rewriting this collection's files. Use this in UI to show a
+    // "compacting…" indicator. Counts both background merges (maybeMerge) and explicit compact/merge
+    // calls. Becomes false once the new index is swapped in (the deferred-delete window is NOT counted).
+    public isCompactingSync(): boolean {
+        return this.reader.isCompactingSync();
+    }
+
     public async getColumnInfo() {
         const index = await this.ensureIndex();
         return index.reader.columns;

package/storage/BulkDatabase2/BulkDatabaseReader.d.ts

@@ -31,6 +31,10 @@ export declare class BulkDatabaseReader<T extends {
     isKeyWatched(key: string): boolean;
     isLiveNow(key: string): boolean;
     localTime(key: string): number;
+    private compactingCount;
+    beginCompaction(): void;
+    endCompaction(): void;
+    isCompactingSync(): boolean;
     private notifyOverlayMutation;
     getKeys(): Promise<string[]>;
     getColumn<C extends keyof T>(column: C): Promise<{

package/storage/BulkDatabase2/BulkDatabaseReader.ts

@@ -7,6 +7,7 @@ import { blue, red } from "socket-function/src/formatting/logColors";
 const NULL = String.fromCharCode(0);
 const LOAD_SIGNAL = NULL + "load";
 const OVERLAY_SIGNAL = NULL + "overlay";
+const COMPACTING_SIGNAL = NULL + "compacting";
 const TRIGGER_THROTTLE_FIRST_STEP_MS = 16;
 
 function nullJoin(a: string, b: string): string {
@@ -88,6 +89,22 @@ export class BulkDatabaseReader<T extends { key: string }> {
         return -Infinity;
     }
 
+    // Counter (so concurrent / nested merges in the same instance compose correctly). Signal fires
+    // immediately, not through the trigger throttle — UI spinners should show right away.
+    private compactingCount = 0;
+    beginCompaction(): void {
+        this.compactingCount++;
+        if (this.compactingCount === 1) this.cfg.deps.invalidate(COMPACTING_SIGNAL);
+    }
+    endCompaction(): void {
+        this.compactingCount--;
+        if (this.compactingCount === 0) this.cfg.deps.invalidate(COMPACTING_SIGNAL);
+    }
+    isCompactingSync(): boolean {
+        this.cfg.deps.observe(COMPACTING_SIGNAL);
+        return this.compactingCount > 0;
+    }
+
     private notifyOverlayMutation(key: string, columns: Iterable<string> | "all"): void {
         this.dataGen++;
         if (columns === "all") this.columnCache.clear();