sliftutils 1.4.43 → 1.4.45

package/index.d.ts

@@ -912,6 +912,8 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         firstMergeTriggerRangeMs: number;
         streamFoldTriggerRows: number;
         streamFoldTriggerBytes: number;
+        streamFileMaxBytes: number;
+        streamFoldHardLimitBytes: number;
         writeFlushMaxDelayMs: number;
         fileSetPollIntervalMs: number;
     };
@@ -965,6 +967,8 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private streamRowsOnDisk;
         private streamBytesOnDisk;
         private streamFileName;
+        private currentStreamFileName;
+        private currentStreamFileBytes;
         private lastMergeCheck;
         private getStreamFileName;
         private invalidateOverlay;
@@ -991,6 +995,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         flush(): Promise<void>;
         private flushPending;
         private doFlush;
+        private foldOwnStream;
         update(entry: Partial<T> & {
             key: string;
         }): Promise<void>;
@@ -1038,6 +1043,8 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private baseColumnsLoading;
         private baseFields;
         private baseFieldsLoading;
+        private staleBaseColumns;
+        private staleBaseFields;
         private ensureBaseColumn;
         private ensureBaseField;
         getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.43",
+    "version": "1.4.45",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -8,6 +8,8 @@ export declare const bulkDatabase2Timing: {
     firstMergeTriggerRangeMs: number;
     streamFoldTriggerRows: number;
     streamFoldTriggerBytes: number;
+    streamFileMaxBytes: number;
+    streamFoldHardLimitBytes: number;
     writeFlushMaxDelayMs: number;
     fileSetPollIntervalMs: number;
 };
@@ -61,6 +63,8 @@ export declare class BulkDatabaseBase<T extends {
     private streamRowsOnDisk;
     private streamBytesOnDisk;
     private streamFileName;
+    private currentStreamFileName;
+    private currentStreamFileBytes;
     private lastMergeCheck;
     private getStreamFileName;
     private invalidateOverlay;
@@ -87,6 +91,7 @@ export declare class BulkDatabaseBase<T extends {
     flush(): Promise<void>;
     private flushPending;
     private doFlush;
+    private foldOwnStream;
     update(entry: Partial<T> & {
         key: string;
     }): Promise<void>;
@@ -134,6 +139,8 @@ export declare class BulkDatabaseBase<T extends {
     private baseColumnsLoading;
     private baseFields;
     private baseFieldsLoading;
+    private staleBaseColumns;
+    private staleBaseFields;
     private ensureBaseColumn;
     private ensureBaseField;
     getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -86,6 +86,16 @@ export const bulkDatabase2Timing = {
     // throttle. The byte bound matters most over the network, where pulling the whole stream is expensive.
     streamFoldTriggerRows: 100,
     streamFoldTriggerBytes: 64 * 1024 * 1024,
+    // Rolling cap on OUR OWN current stream file: once we've appended this much to it, we seal it (stop
+    // writing) and fold it into bulk immediately. Without this, many small appends grow a single stream
+    // file without bound (the per-batch ROLLOVER only catches one huge batch). We can fold our own file at
+    // once — the 10h seal-age rule is only for other writers' files; we know we're done with ours on seal.
+    streamFileMaxBytes: 50 * 1024 * 1024,
+    // HARD limit: once the stream tier passes this, fold it NOW regardless of age (even un-sealed, recent
+    // streams) — a stream this large makes every read pull a huge file, i.e. the collection becomes
+    // essentially unreadable. We still only delete a stream whose size didn't change while we read it, so
+    // a writer mid-append never loses data (it just gets re-folded next pass).
+    streamFoldHardLimitBytes: 768 * 1024 * 1024,
     // Max delay (per collection) before buffered stream writes are flushed to disk; the delay ramps up to
     // this under sustained writing (see WRITE_FLUSH_FIRST_STEP_MS). 0 = flush every write immediately —
     // the default in Node, where append is real and cheap; the browser ramps to 15s to avoid rewriting
@@ -369,6 +379,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // This instance's tier-0 stream file. Each instance (≈ each thread/tab) streams to its own file
     // so concurrent writers never touch the same file.
     private streamFileName: string | undefined;
+    // Rolling size of the current stream file, so we can seal+fold it once it passes STREAM_FILE_MAX_BYTES
+    // (reset whenever the file rotates). currentStreamFileName tracks which file the count belongs to.
+    private currentStreamFileName: string | undefined;
+    private currentStreamFileBytes = 0;
     // Seeded to construction time (not 0) so a fresh instance doesn't immediately seal+merge on its very
     // first write — the first background merge check waits a full interval after construction.
     private lastMergeCheck = Date.now();
@@ -563,6 +577,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // buildReader. Does NOT touch the overlay — callers decide what to do with pending writes.
     private clearReaderState() {
         this.reader.reset();
+        // Preserve the last-known sync base values so a reload/compact serves them (not empty) while the
+        // fresh ones reload — observers transition old → new, never flashing through nothing.
+        for (const [k, v] of this.baseColumns) this.staleBaseColumns.set(k, v);
+        for (const [k, v] of this.baseFields) this.staleBaseFields.set(k, v);
         this.baseColumns.clear();
         this.baseColumnsLoading.clear();
         this.baseFields.clear();
@@ -726,13 +744,41 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const batch = this.pendingAppends.slice();
         const combined = Buffer.concat(batch.map(p => p.framed));
         const storage = await this.storage();
+        const fileName = this.getStreamFileName();
+        if (fileName !== this.currentStreamFileName) { // rotated (age-seal / first write) → reset the count
+            this.currentStreamFileName = fileName;
+            this.currentStreamFileBytes = 0;
+        }
         // Throws on failure -> we don't splice, so the data stays buffered and a later flush retries it.
-        await storage.append(this.getStreamFileName(), combined);
+        await storage.append(fileName, combined);
         // New writes added during the await are after `batch`, so removing the front is exactly the flushed set.
         this.pendingAppends.splice(0, batch.length);
         // Keep the stream-size estimate current so the fold trigger sees write-accumulation between reads.
         this.streamBytesOnDisk += combined.length;
         for (const p of batch) this.streamRowsOnDisk += p.rows;
+        this.currentStreamFileBytes += combined.length;
+        // Cap our own stream file: once it's grown past the limit, seal it (next write starts a fresh file)
+        // and fold this now-complete file into bulk in the background. New writes go to the new file, so the
+        // sealed one is stable and safe to fold+delete immediately — no 10h wait for our own files.
+        if (this.currentStreamFileBytes >= bulkDatabase2Timing.streamFileMaxBytes) {
+            this.streamFileName = undefined;
+            this.currentStreamFileName = undefined;
+            this.currentStreamFileBytes = 0;
+            void this.foldOwnStream(fileName);
+        }
+    }
+
+    // Fold one of OUR OWN sealed stream files into bulk and delete it. Safe to delete immediately (force):
+    // we've stopped appending to it (a new file is current now), and canDeleteStream still only deletes it
+    // if its size is unchanged since we read it, so an in-flight flush can never lose data.
+    private async foldOwnStream(fileName: string): Promise<void> {
+        const info = parseStreamFileName(fileName);
+        if (!info) return;
+        try {
+            await this.mergeFileSet([], [info], false, true);
+        } catch (e) {
+            console.warn(`${this.name}: folding own stream ${fileName} failed: ${(e as Error).message}`);
+        }
     }
 
     public async update(entry: Partial<T> & { key: string }): Promise<void> {
@@ -1093,7 +1139,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // outside the merge; if nothing older exists, that older set can't exist either, so the tombstone has
     // nothing left to suppress and we drop it instead of carrying it forward. (A full compact and a
     // merge that reaches time 0 are the cases where this holds.)
-    private async mergeFileSet(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest = false): Promise<boolean> {
+    private async mergeFileSet(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest = false, forceDeleteStreams = false): Promise<boolean> {
         const storage = await this.storage();
         const timestamp = nextFileTime();
         const now = Date.now();
@@ -1155,7 +1201,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const remove = async (name: string) => { try { await storage.remove(name); } catch { /* already gone */ } };
         for (const f of consumedBulk) await remove(f.fileName);
         for (const f of streamFiles) {
-            if (await this.canDeleteStream(f, now, streamData.sizes)) await remove(f.fileName);
+            if (await this.canDeleteStream(f, now, streamData.sizes, forceDeleteStreams)) await remove(f.fileName);
         }
 
         this.resetReader();
@@ -1168,9 +1214,12 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // the merge). When neither holds we leave it: its data is now duplicated into bulk (resolved by time)
     // and a later merge deletes it once aged. (Recreate-on-append means even a wrong delete wouldn't lose
     // data, but the aged check also rules out the rare sparse-offset append race.)
-    private async canDeleteStream(f: StreamFileInfo, now: number, sizes: Map<string, number>): Promise<boolean> {
+    private async canDeleteStream(f: StreamFileInfo, now: number, sizes: Map<string, number>, force = false): Promise<boolean> {
         if (now - f.timestamp >= bulkDatabase2Timing.streamSealAgeMs) return true;
-        if (!isSyncSupported()) return false;
+        // Without cross-tab sync we normally can't tell a writer is done before the seal age — UNLESS the
+        // caller forces it (the hard stream limit). Even then we only delete a stream whose size didn't
+        // change while we read it, so a writer mid-append never loses data (it's re-folded next pass).
+        if (!isSyncSupported() && !force) return false;
         const readSize = sizes.get(f.fileName);
         if (readSize === undefined) return false;
         let info;
@@ -1217,6 +1266,23 @@ export class BulkDatabaseBase<T extends { key: string }> {
             return true;
         };
 
+        // ── Hard stream limit: once the tier-0 stream has grown past the hard cap, fold ALL of it into bulk
+        //    NOW regardless of age — a stream this big makes every read pull a huge file, i.e. the collection
+        //    is essentially unreadable. Force-delete the folded streams (canDeleteStream still only deletes
+        //    size-stable ones, so an active writer never loses data; it's just re-folded next pass). ──
+        {
+            const { streamFiles } = await this.listFiles();
+            if (streamFiles.length) {
+                const storage = await this.storage();
+                const sizes = await Promise.all(streamFiles.map(async f => { try { return (await storage.getInfo(f.fileName))?.size ?? 0; } catch { return 0; } }));
+                const totalStreamBytes = sizes.reduce((a, b) => a + b, 0);
+                if (totalStreamBytes > bulkDatabase2Timing.streamFoldHardLimitBytes) {
+                    console.log(`${blue(this.name)} stream tier ${fmtBytes(totalStreamBytes)} over hard limit ${fmtBytes(bulkDatabase2Timing.streamFoldHardLimitBytes)} — folding all streams now`);
+                    if (await this.mergeFileSet([], streamFiles, false, true)) merged = true;
+                }
+            }
+        }
+
         // ── Pass 1: consolidate recent files. ──
         // Only seal (ask peers + ourselves to abandon current stream files) when cross-tab sync can
         // actually fold recent streams; in Node it would just churn — fragmenting streams every pass for
@@ -1249,12 +1315,16 @@ export class BulkDatabaseBase<T extends { key: string }> {
                 if (recentBytes >= FIRST_MERGE_BYTES) break;
             }
             const span = recent.length ? recent[0].time - recent[recent.length - 1].time : 0;
-            // Fold when there's enough fragmentation to consolidate, OR when a foldable stream has grown too
-            // big to keep reading whole (even if it's the only recent file) — see streamNeedsFold.
-            const heavyStreamRecent = this.streamNeedsFold() && recent.some(i => i.kind === "stream");
+            // Fold when there's enough fragmentation to consolidate, OR when the foldable stream data here
+            // has grown past the byte threshold (stream data can't be read per-cell, so a big stream is
+            // costly to pull whole). The stream size is derived from THIS fresh listing — not a cached
+            // counter — so it's correct even for an instance that never built a reader (e.g. the host's
+            // autocompactor) and for streams that grew in place since the index was last loaded.
+            const recentStreamBytes = recent.reduce((a, it) => a + (it.kind === "stream" ? it.bytes : 0), 0);
+            const heavyStream = recentStreamBytes > bulkDatabase2Timing.streamFoldTriggerBytes;
             const triggered =
                 recent.length >= 2 && (recent.length > bulkDatabase2Timing.firstMergeTriggerFiles || span > bulkDatabase2Timing.firstMergeTriggerRangeMs)
-                || heavyStreamRecent && recent.length >= 1;
+                || heavyStream;
             if (triggered) {
                 const rb = recent.filter(i => i.kind === "bulk").map(i => (i.file as BulkFileInfo));
                 const rs = recent.filter(i => i.kind === "stream").map(i => (i.file as StreamFileInfo));
@@ -1416,6 +1486,11 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // (Map.has distinguishes "loaded" from "not loaded yet").
     private baseFields = new Map<string, { value: unknown; time: number } | undefined>();
     private baseFieldsLoading = new Set<string>();
+    // Last-known sync base values, kept across a reader reset so a reload/compact serves the previous data
+    // instead of flashing empty while the fresh value reloads in the background. Each entry is dropped once
+    // ensureBase* has the corresponding fresh value (so reads transition old → new, never old → empty → new).
+    private staleBaseColumns = new Map<string, { key: string; value: unknown; time: number }[]>();
+    private staleBaseFields = new Map<string, { value: unknown; time: number } | undefined>();
 
     private ensureBaseColumn(column: string) {
         if (this.baseColumns.has(column) || this.baseColumnsLoading.has(column)) return;
@@ -1425,6 +1500,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
                 const base = await this.readWithReload(reader => reader.getColumn(column));
                 this.deps.batch(() => {
                     this.baseColumns.set(column, base);
+                    this.staleBaseColumns.delete(column); // fresh value in hand; stop serving the stale one
                     this.baseColumnsLoading.delete(column);
                     this.invalidateSignal(LOAD_SIGNAL);
                 });
@@ -1446,6 +1522,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
                 const resolved = await this.readWithReload(reader => reader.getSingleField(key, column));
                 this.deps.batch(() => {
                     this.baseFields.set(cacheKey, resolved);
+                    this.staleBaseFields.delete(cacheKey); // fresh value in hand; stop serving the stale one
                     this.baseFieldsLoading.delete(cacheKey);
                     this.invalidateSignal(LOAD_SIGNAL);
                 });
@@ -1470,15 +1547,30 @@ export class BulkDatabaseBase<T extends { key: string }> {
         let entry = this.overlay.get(key);
         if (entry !== undefined) {
             if (entry.value === DELETED) return undefined;
-            if (col in entry.value) return { key, value: entry.value[col] as T[Column], time: entry.time };
+            if (col in entry.value) {
+                // Warm the disk-backed base in the background even though the overlay serves this now — so
+                // when the overlay is later cleared (e.g. compaction persists it), there's a value to serve
+                // and the read doesn't flash empty.
+                this.ensureBaseField(key, col);
+                return { key, value: entry.value[col] as T[Column], time: entry.time };
+            }
             // column not set in the overlay entry — fall through to the base field cache for this column
         }
         let cacheKey = nullJoin(col, key);
-        if (!this.baseFields.has(cacheKey)) {
+        // Use the fresh value if loaded; mid-reload fall back to the last-known one so we don't flash empty.
+        let src: Map<string, { value: unknown; time: number } | undefined> | undefined;
+        if (this.baseFields.has(cacheKey)) {
+            src = this.baseFields;
+        } else {
             this.ensureBaseField(key, col);
+            src = this.staleBaseFields.has(cacheKey) ? this.staleBaseFields : undefined;
+        }
+        if (!src) {
+            // Genuine first load (nothing known); but an overlay entry makes the key live with this column unset.
+            if (entry !== undefined && entry.value !== DELETED) return { key, value: undefined as T[Column], time: entry.time };
             return undefined;
         }
-        const base = this.baseFields.get(cacheKey);
+        const base = src.get(cacheKey);
         if (base === undefined) {
             // Not live on disk; but an overlay entry for the key (partial write) makes it live, column unset.
             if (entry !== undefined && entry.value !== DELETED) return { key, value: undefined as T[Column], time: entry.time };
@@ -1498,7 +1590,11 @@ export class BulkDatabaseBase<T extends { key: string }> {
         let base = this.baseColumns.get(col);
         if (!base) {
             this.ensureBaseColumn(col);
-            return undefined;
+            // Mid-reload: serve the last-known value (patched with the current overlay) so we don't flash
+            // empty. Don't cache it — it's stale until ensureBaseColumn swaps in the fresh value.
+            const stale = this.staleBaseColumns.get(col);
+            if (stale) return this.patchColumn(stale, col) as { key: string; value: T[Column]; time: number }[];
+            return undefined; // genuine first load — nothing known yet
         }
         // Synchronous (no awaits) → reads the current overlay and caches it atomically; an observer re-runs
         // (and the cache is cleared) on any later change, so the frozen result is safe to share.