sliftutils 1.4.44 → 1.4.46

package/index.d.ts

@@ -845,6 +845,14 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
             value: T[Column];
             time: number;
         }[] | undefined;
+        /**
+         * Reactive: whether (key, column) is loaded yet — true once we know the answer (value, absent, or
+         * deleted), false while it's still loading. getSingleFieldObjSync returns undefined for BOTH "loading"
+         * and "absent", so use this to tell them apart (e.g. show a spinner only when this is false).
+         */
+        isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
+        /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
+        isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
         /** The columns present on disk and their byte sizes (no row data read). */
         getColumnInfo(): Promise<BulkColumnInfo[]>;
         /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
@@ -912,6 +920,8 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         firstMergeTriggerRangeMs: number;
         streamFoldTriggerRows: number;
         streamFoldTriggerBytes: number;
+        streamFileMaxBytes: number;
+        streamFoldHardLimitBytes: number;
         writeFlushMaxDelayMs: number;
         fileSetPollIntervalMs: number;
     };
@@ -965,6 +975,8 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private streamRowsOnDisk;
         private streamBytesOnDisk;
         private streamFileName;
+        private currentStreamFileName;
+        private currentStreamFileBytes;
         private lastMergeCheck;
         private getStreamFileName;
         private invalidateOverlay;
@@ -991,6 +1003,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         flush(): Promise<void>;
         private flushPending;
         private doFlush;
+        private foldOwnStream;
         update(entry: Partial<T> & {
             key: string;
         }): Promise<void>;
@@ -1038,6 +1051,8 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private baseColumnsLoading;
         private baseFields;
         private baseFieldsLoading;
+        private staleBaseColumns;
+        private staleBaseFields;
         private ensureBaseColumn;
         private ensureBaseField;
         getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
@@ -1051,6 +1066,8 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
             value: T[Column];
             time: number;
         }[] | undefined;
+        isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
+        isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
         getColumnInfo(): Promise<{
             column: string;
             byteSize: number;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.44",
+    "version": "1.4.46",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabase2.d.ts

@@ -80,6 +80,14 @@ export interface IBulkDatabase2<T extends {
         value: T[Column];
         time: number;
     }[] | undefined;
+    /**
+     * Reactive: whether (key, column) is loaded yet — true once we know the answer (value, absent, or
+     * deleted), false while it's still loading. getSingleFieldObjSync returns undefined for BOTH "loading"
+     * and "absent", so use this to tell them apart (e.g. show a spinner only when this is false).
+     */
+    isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
+    /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
+    isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
     /** The columns present on disk and their byte sizes (no row data read). */
     getColumnInfo(): Promise<BulkColumnInfo[]>;
     /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */

package/storage/BulkDatabase2/BulkDatabase2.ts

@@ -65,6 +65,15 @@ export interface IBulkDatabase2<T extends { key: string }> {
     /** Synchronous, reactive read of a whole column ({ key, value, time }). undefined while still loading. */
     getColumnSync<Column extends keyof T>(column: Column): { key: string; value: T[Column]; time: number }[] | undefined;
 
+    /**
+     * Reactive: whether (key, column) is loaded yet — true once we know the answer (value, absent, or
+     * deleted), false while it's still loading. getSingleFieldObjSync returns undefined for BOTH "loading"
+     * and "absent", so use this to tell them apart (e.g. show a spinner only when this is false).
+     */
+    isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
+    /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
+    isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
+
     /** The columns present on disk and their byte sizes (no row data read). */
     getColumnInfo(): Promise<BulkColumnInfo[]>;
     /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -8,6 +8,8 @@ export declare const bulkDatabase2Timing: {
     firstMergeTriggerRangeMs: number;
     streamFoldTriggerRows: number;
     streamFoldTriggerBytes: number;
+    streamFileMaxBytes: number;
+    streamFoldHardLimitBytes: number;
     writeFlushMaxDelayMs: number;
     fileSetPollIntervalMs: number;
 };
@@ -61,6 +63,8 @@ export declare class BulkDatabaseBase<T extends {
     private streamRowsOnDisk;
     private streamBytesOnDisk;
     private streamFileName;
+    private currentStreamFileName;
+    private currentStreamFileBytes;
     private lastMergeCheck;
     private getStreamFileName;
     private invalidateOverlay;
@@ -87,6 +91,7 @@ export declare class BulkDatabaseBase<T extends {
     flush(): Promise<void>;
     private flushPending;
     private doFlush;
+    private foldOwnStream;
     update(entry: Partial<T> & {
         key: string;
     }): Promise<void>;
@@ -134,6 +139,8 @@ export declare class BulkDatabaseBase<T extends {
     private baseColumnsLoading;
     private baseFields;
     private baseFieldsLoading;
+    private staleBaseColumns;
+    private staleBaseFields;
     private ensureBaseColumn;
     private ensureBaseField;
     getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
@@ -147,6 +154,8 @@ export declare class BulkDatabaseBase<T extends {
         value: T[Column];
         time: number;
     }[] | undefined;
+    isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
+    isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
     getColumnInfo(): Promise<{
         column: string;
         byteSize: number;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -86,6 +86,16 @@ export const bulkDatabase2Timing = {
     // throttle. The byte bound matters most over the network, where pulling the whole stream is expensive.
     streamFoldTriggerRows: 100,
     streamFoldTriggerBytes: 64 * 1024 * 1024,
+    // Rolling cap on OUR OWN current stream file: once we've appended this much to it, we seal it (stop
+    // writing) and fold it into bulk immediately. Without this, many small appends grow a single stream
+    // file without bound (the per-batch ROLLOVER only catches one huge batch). We can fold our own file at
+    // once — the 10h seal-age rule is only for other writers' files; we know we're done with ours on seal.
+    streamFileMaxBytes: 50 * 1024 * 1024,
+    // HARD limit: once the stream tier passes this, fold it NOW regardless of age (even un-sealed, recent
+    // streams) — a stream this large makes every read pull a huge file, i.e. the collection becomes
+    // essentially unreadable. We still only delete a stream whose size didn't change while we read it, so
+    // a writer mid-append never loses data (it just gets re-folded next pass).
+    streamFoldHardLimitBytes: 768 * 1024 * 1024,
     // Max delay (per collection) before buffered stream writes are flushed to disk; the delay ramps up to
     // this under sustained writing (see WRITE_FLUSH_FIRST_STEP_MS). 0 = flush every write immediately —
     // the default in Node, where append is real and cheap; the browser ramps to 15s to avoid rewriting
@@ -369,6 +379,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // This instance's tier-0 stream file. Each instance (≈ each thread/tab) streams to its own file
     // so concurrent writers never touch the same file.
     private streamFileName: string | undefined;
+    // Rolling size of the current stream file, so we can seal+fold it once it passes STREAM_FILE_MAX_BYTES
+    // (reset whenever the file rotates). currentStreamFileName tracks which file the count belongs to.
+    private currentStreamFileName: string | undefined;
+    private currentStreamFileBytes = 0;
     // Seeded to construction time (not 0) so a fresh instance doesn't immediately seal+merge on its very
     // first write — the first background merge check waits a full interval after construction.
     private lastMergeCheck = Date.now();
@@ -563,6 +577,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // buildReader. Does NOT touch the overlay — callers decide what to do with pending writes.
     private clearReaderState() {
         this.reader.reset();
+        // Preserve the last-known sync base values so a reload/compact serves them (not empty) while the
+        // fresh ones reload — observers transition old → new, never flashing through nothing.
+        for (const [k, v] of this.baseColumns) this.staleBaseColumns.set(k, v);
+        for (const [k, v] of this.baseFields) this.staleBaseFields.set(k, v);
         this.baseColumns.clear();
         this.baseColumnsLoading.clear();
         this.baseFields.clear();
@@ -726,13 +744,41 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const batch = this.pendingAppends.slice();
         const combined = Buffer.concat(batch.map(p => p.framed));
         const storage = await this.storage();
+        const fileName = this.getStreamFileName();
+        if (fileName !== this.currentStreamFileName) { // rotated (age-seal / first write) → reset the count
+            this.currentStreamFileName = fileName;
+            this.currentStreamFileBytes = 0;
+        }
         // Throws on failure -> we don't splice, so the data stays buffered and a later flush retries it.
-        await storage.append(this.getStreamFileName(), combined);
+        await storage.append(fileName, combined);
         // New writes added during the await are after `batch`, so removing the front is exactly the flushed set.
         this.pendingAppends.splice(0, batch.length);
         // Keep the stream-size estimate current so the fold trigger sees write-accumulation between reads.
         this.streamBytesOnDisk += combined.length;
         for (const p of batch) this.streamRowsOnDisk += p.rows;
+        this.currentStreamFileBytes += combined.length;
+        // Cap our own stream file: once it's grown past the limit, seal it (next write starts a fresh file)
+        // and fold this now-complete file into bulk in the background. New writes go to the new file, so the
+        // sealed one is stable and safe to fold+delete immediately — no 10h wait for our own files.
+        if (this.currentStreamFileBytes >= bulkDatabase2Timing.streamFileMaxBytes) {
+            this.streamFileName = undefined;
+            this.currentStreamFileName = undefined;
+            this.currentStreamFileBytes = 0;
+            void this.foldOwnStream(fileName);
+        }
+    }
+
+    // Fold one of OUR OWN sealed stream files into bulk and delete it. Safe to delete immediately (force):
+    // we've stopped appending to it (a new file is current now), and canDeleteStream still only deletes it
+    // if its size is unchanged since we read it, so an in-flight flush can never lose data.
+    private async foldOwnStream(fileName: string): Promise<void> {
+        const info = parseStreamFileName(fileName);
+        if (!info) return;
+        try {
+            await this.mergeFileSet([], [info], false, true);
+        } catch (e) {
+            console.warn(`${this.name}: folding own stream ${fileName} failed: ${(e as Error).message}`);
+        }
     }
 
     public async update(entry: Partial<T> & { key: string }): Promise<void> {
@@ -1093,7 +1139,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // outside the merge; if nothing older exists, that older set can't exist either, so the tombstone has
     // nothing left to suppress and we drop it instead of carrying it forward. (A full compact and a
     // merge that reaches time 0 are the cases where this holds.)
-    private async mergeFileSet(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest = false): Promise<boolean> {
+    private async mergeFileSet(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest = false, forceDeleteStreams = false): Promise<boolean> {
         const storage = await this.storage();
         const timestamp = nextFileTime();
         const now = Date.now();
@@ -1155,7 +1201,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const remove = async (name: string) => { try { await storage.remove(name); } catch { /* already gone */ } };
         for (const f of consumedBulk) await remove(f.fileName);
         for (const f of streamFiles) {
-            if (await this.canDeleteStream(f, now, streamData.sizes)) await remove(f.fileName);
+            if (await this.canDeleteStream(f, now, streamData.sizes, forceDeleteStreams)) await remove(f.fileName);
         }
 
         this.resetReader();
@@ -1168,9 +1214,12 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // the merge). When neither holds we leave it: its data is now duplicated into bulk (resolved by time)
     // and a later merge deletes it once aged. (Recreate-on-append means even a wrong delete wouldn't lose
     // data, but the aged check also rules out the rare sparse-offset append race.)
-    private async canDeleteStream(f: StreamFileInfo, now: number, sizes: Map<string, number>): Promise<boolean> {
+    private async canDeleteStream(f: StreamFileInfo, now: number, sizes: Map<string, number>, force = false): Promise<boolean> {
         if (now - f.timestamp >= bulkDatabase2Timing.streamSealAgeMs) return true;
-        if (!isSyncSupported()) return false;
+        // Without cross-tab sync we normally can't tell a writer is done before the seal age — UNLESS the
+        // caller forces it (the hard stream limit). Even then we only delete a stream whose size didn't
+        // change while we read it, so a writer mid-append never loses data (it's re-folded next pass).
+        if (!isSyncSupported() && !force) return false;
         const readSize = sizes.get(f.fileName);
         if (readSize === undefined) return false;
         let info;
@@ -1217,6 +1266,23 @@ export class BulkDatabaseBase<T extends { key: string }> {
             return true;
         };
 
+        // ── Hard stream limit: once the tier-0 stream has grown past the hard cap, fold ALL of it into bulk
+        //    NOW regardless of age — a stream this big makes every read pull a huge file, i.e. the collection
+        //    is essentially unreadable. Force-delete the folded streams (canDeleteStream still only deletes
+        //    size-stable ones, so an active writer never loses data; it's just re-folded next pass). ──
+        {
+            const { streamFiles } = await this.listFiles();
+            if (streamFiles.length) {
+                const storage = await this.storage();
+                const sizes = await Promise.all(streamFiles.map(async f => { try { return (await storage.getInfo(f.fileName))?.size ?? 0; } catch { return 0; } }));
+                const totalStreamBytes = sizes.reduce((a, b) => a + b, 0);
+                if (totalStreamBytes > bulkDatabase2Timing.streamFoldHardLimitBytes) {
+                    console.log(`${blue(this.name)} stream tier ${fmtBytes(totalStreamBytes)} over hard limit ${fmtBytes(bulkDatabase2Timing.streamFoldHardLimitBytes)} — folding all streams now`);
+                    if (await this.mergeFileSet([], streamFiles, false, true)) merged = true;
+                }
+            }
+        }
+
         // ── Pass 1: consolidate recent files. ──
         // Only seal (ask peers + ourselves to abandon current stream files) when cross-tab sync can
         // actually fold recent streams; in Node it would just churn — fragmenting streams every pass for
@@ -1420,6 +1486,11 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // (Map.has distinguishes "loaded" from "not loaded yet").
     private baseFields = new Map<string, { value: unknown; time: number } | undefined>();
     private baseFieldsLoading = new Set<string>();
+    // Last-known sync base values, kept across a reader reset so a reload/compact serves the previous data
+    // instead of flashing empty while the fresh value reloads in the background. Each entry is dropped once
+    // ensureBase* has the corresponding fresh value (so reads transition old → new, never old → empty → new).
+    private staleBaseColumns = new Map<string, { key: string; value: unknown; time: number }[]>();
+    private staleBaseFields = new Map<string, { value: unknown; time: number } | undefined>();
 
     private ensureBaseColumn(column: string) {
         if (this.baseColumns.has(column) || this.baseColumnsLoading.has(column)) return;
@@ -1429,6 +1500,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
                 const base = await this.readWithReload(reader => reader.getColumn(column));
                 this.deps.batch(() => {
                     this.baseColumns.set(column, base);
+                    this.staleBaseColumns.delete(column); // fresh value in hand; stop serving the stale one
                     this.baseColumnsLoading.delete(column);
                     this.invalidateSignal(LOAD_SIGNAL);
                 });
@@ -1450,6 +1522,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
                 const resolved = await this.readWithReload(reader => reader.getSingleField(key, column));
                 this.deps.batch(() => {
                     this.baseFields.set(cacheKey, resolved);
+                    this.staleBaseFields.delete(cacheKey); // fresh value in hand; stop serving the stale one
                     this.baseFieldsLoading.delete(cacheKey);
                     this.invalidateSignal(LOAD_SIGNAL);
                 });
@@ -1474,15 +1547,30 @@ export class BulkDatabaseBase<T extends { key: string }> {
         let entry = this.overlay.get(key);
         if (entry !== undefined) {
             if (entry.value === DELETED) return undefined;
-            if (col in entry.value) return { key, value: entry.value[col] as T[Column], time: entry.time };
+            if (col in entry.value) {
+                // Warm the disk-backed base in the background even though the overlay serves this now — so
+                // when the overlay is later cleared (e.g. compaction persists it), there's a value to serve
+                // and the read doesn't flash empty.
+                this.ensureBaseField(key, col);
+                return { key, value: entry.value[col] as T[Column], time: entry.time };
+            }
             // column not set in the overlay entry — fall through to the base field cache for this column
         }
         let cacheKey = nullJoin(col, key);
-        if (!this.baseFields.has(cacheKey)) {
+        // Use the fresh value if loaded; mid-reload fall back to the last-known one so we don't flash empty.
+        let src: Map<string, { value: unknown; time: number } | undefined> | undefined;
+        if (this.baseFields.has(cacheKey)) {
+            src = this.baseFields;
+        } else {
             this.ensureBaseField(key, col);
+            src = this.staleBaseFields.has(cacheKey) ? this.staleBaseFields : undefined;
+        }
+        if (!src) {
+            // Genuine first load (nothing known); but an overlay entry makes the key live with this column unset.
+            if (entry !== undefined && entry.value !== DELETED) return { key, value: undefined as T[Column], time: entry.time };
             return undefined;
         }
-        const base = this.baseFields.get(cacheKey);
+        const base = src.get(cacheKey);
         if (base === undefined) {
             // Not live on disk; but an overlay entry for the key (partial write) makes it live, column unset.
             if (entry !== undefined && entry.value !== DELETED) return { key, value: undefined as T[Column], time: entry.time };
@@ -1502,7 +1590,11 @@ export class BulkDatabaseBase<T extends { key: string }> {
         let base = this.baseColumns.get(col);
         if (!base) {
             this.ensureBaseColumn(col);
-            return undefined;
+            // Mid-reload: serve the last-known value (patched with the current overlay) so we don't flash
+            // empty. Don't cache it — it's stale until ensureBaseColumn swaps in the fresh value.
+            const stale = this.staleBaseColumns.get(col);
+            if (stale) return this.patchColumn(stale, col) as { key: string; value: T[Column]; time: number }[];
+            return undefined; // genuine first load — nothing known yet
         }
         // Synchronous (no awaits) → reads the current overlay and caches it atomically; an observer re-runs
         // (and the cache is cleared) on any later change, so the frozen result is safe to share.
@@ -1512,6 +1604,38 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return result;
     }
 
+    // Reactive: whether (key, column) is available to read synchronously yet. true once it's loaded — we
+    // know the answer, whether that's a value, absent, or deleted; false while it's still loading from disk.
+    // Pairs with getSingleFieldObjSync, which returns undefined for BOTH "loading" and "absent" — use this
+    // to tell them apart (e.g. show a spinner only when this is false). Triggers the load if not started,
+    // and (like the sync reads) counts the last-known value served during a reload as loaded.
+    public isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean {
+        void this.syncSetup();
+        this.deps.observe(LOAD_SIGNAL);
+        this.deps.observe(key);
+        const entry = this.overlay.get(key);
+        if (entry !== undefined) {
+            if (entry.value === DELETED) return true;          // known: deleted
+            if (String(column) in entry.value) return true;    // known: overlay holds this column
+            // else: this column falls through to disk — check the base caches below
+        }
+        const cacheKey = nullJoin(String(column), key);
+        if (this.baseFields.has(cacheKey) || this.staleBaseFields.has(cacheKey)) return true;
+        this.ensureBaseField(key, String(column));
+        return false;
+    }
+
+    // Reactive: whether a whole column is available to read synchronously yet (see isFieldLoadedSync).
+    public isColumnLoadedSync<Column extends keyof T>(column: Column): boolean {
+        void this.syncSetup();
+        this.deps.observe(LOAD_SIGNAL);
+        this.deps.observe(OVERLAY_SIGNAL);
+        const col = String(column);
+        if (this.columnCache.has(col) || this.baseColumns.has(col) || this.staleBaseColumns.has(col)) return true;
+        this.ensureBaseColumn(col);
+        return false;
+    }
+
     public async getColumnInfo() {
         let reader = await this.reader();
         return reader.columns;