sliftutils 1.4.14 → 1.4.16

package/index.d.ts

@@ -865,6 +865,13 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
         }>;
         /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
         compact(): Promise<void>;
+        /**
+         * Whether this collection's storage is served over the network (a remote server) rather than local
+         * disk. Apps can branch on this to adapt to the higher latency. Note: over the network the database
+         * skips automatic background compaction by default — call the static
+         * `BulkDatabase2.enableNetworkCompaction()` once to opt in.
+         */
+        isRemote(): Promise<boolean>;
         /**
          * Flush buffered stream writes to disk now. Writes are coalesced and flushed on a ramping delay (to
          * avoid the browser rewriting the whole stream file per write), so a write's promise resolving means
@@ -902,6 +909,8 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         mergeSpacingMs: number;
         firstMergeTriggerFiles: number;
         firstMergeTriggerRangeMs: number;
+        streamFoldTriggerRows: number;
+        streamFoldTriggerBytes: number;
         writeFlushMaxDelayMs: number;
     };
     export interface ReactiveDeps {
@@ -924,13 +933,21 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private currentFlushDelay;
         private lastWriteTime;
         static clearCache(): void;
+        static enableNetworkCompaction(): void;
         storage: {
             (): Promise<FileStorage>;
             reset(): void;
             set(newValue: Promise<FileStorage>): void;
         };
+        isRemote(): Promise<boolean>;
+        private streamNeedsFold;
+        private automaticCompactionAllowed;
         private overlay;
         private streamTimes;
+        private columnCache;
+        private dataGen;
+        private streamRowsOnDisk;
+        private streamBytesOnDisk;
         private streamFileName;
         private lastMergeCheck;
         private getStreamFileName;
@@ -1372,6 +1389,7 @@ declare module "sliftutils/storage/FileFolderAPI" {
         readonly kind: "directory";
         readonly name: string;
         readonly fullPath?: string;
+        readonly isRemote?: boolean;
         removeEntry(key: string, options?: {
             recursive?: boolean;
         }): Promise<void>;
@@ -1459,6 +1477,7 @@ declare module "sliftutils/storage/FileFolderAPI" {
     };
     export type FileStorage = IStorageRaw & {
         folder: NestedFileStorage;
+        isRemote?: boolean;
     };
     export declare function wrapHandle(handle: DirectoryWrapper): FileStorage;
     export declare function getRemoteFileStorageFactory(url: string, password: string, options?: RemoteOptions): (pathStr: string) => Promise<FileStorage>;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.14",
+    "version": "1.4.16",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabase2.d.ts

@@ -100,6 +100,13 @@ export interface IBulkDatabase2<T extends {
     }>;
     /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
     compact(): Promise<void>;
+    /**
+     * Whether this collection's storage is served over the network (a remote server) rather than local
+     * disk. Apps can branch on this to adapt to the higher latency. Note: over the network the database
+     * skips automatic background compaction by default — call the static
+     * `BulkDatabase2.enableNetworkCompaction()` once to opt in.
+     */
+    isRemote(): Promise<boolean>;
     /**
      * Flush buffered stream writes to disk now. Writes are coalesced and flushed on a ramping delay (to
      * avoid the browser rewriting the whole stream file per write), so a write's promise resolving means

package/storage/BulkDatabase2/BulkDatabase2.ts

@@ -79,6 +79,14 @@ export interface IBulkDatabase2<T extends { key: string }> {
     /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
     compact(): Promise<void>;
 
+    /**
+     * Whether this collection's storage is served over the network (a remote server) rather than local
+     * disk. Apps can branch on this to adapt to the higher latency. Note: over the network the database
+     * skips automatic background compaction by default — call the static
+     * `BulkDatabase2.enableNetworkCompaction()` once to opt in.
+     */
+    isRemote(): Promise<boolean>;
+
     /**
      * Flush buffered stream writes to disk now. Writes are coalesced and flushed on a ramping delay (to
      * avoid the browser rewriting the whole stream file per write), so a write's promise resolving means

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -5,6 +5,8 @@ export declare const bulkDatabase2Timing: {
     mergeSpacingMs: number;
     firstMergeTriggerFiles: number;
     firstMergeTriggerRangeMs: number;
+    streamFoldTriggerRows: number;
+    streamFoldTriggerBytes: number;
     writeFlushMaxDelayMs: number;
 };
 export interface ReactiveDeps {
@@ -27,13 +29,21 @@ export declare class BulkDatabaseBase<T extends {
     private currentFlushDelay;
     private lastWriteTime;
     static clearCache(): void;
+    static enableNetworkCompaction(): void;
     storage: {
         (): Promise<FileStorage>;
         reset(): void;
         set(newValue: Promise<FileStorage>): void;
     };
+    isRemote(): Promise<boolean>;
+    private streamNeedsFold;
+    private automaticCompactionAllowed;
     private overlay;
     private streamTimes;
+    private columnCache;
+    private dataGen;
+    private streamRowsOnDisk;
+    private streamBytesOnDisk;
     private streamFileName;
     private lastMergeCheck;
     private getStreamFileName;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -77,6 +77,11 @@ export const bulkDatabase2Timing = {
     firstMergeTriggerFiles: 20,
     // ...or span a wider write-time range than this (data trickling in slowly still gets consolidated).
     firstMergeTriggerRangeMs: 3 * 24 * 60 * 60 * 1000,
+    // Tier-0 stream data can't be read per-cell — a read pulls the whole stream — so once the stream
+    // exceeds BOTH of these it's folded into bulk (columnar, range-readable) promptly, bypassing the merge
+    // throttle. The byte bound matters most over the network, where pulling the whole stream is expensive.
+    streamFoldTriggerRows: 100,
+    streamFoldTriggerBytes: 64 * 1024 * 1024,
     // Max delay (per collection) before buffered stream writes are flushed to disk; the delay ramps up to
     // this under sustained writing (see WRITE_FLUSH_FIRST_STEP_MS). 0 = flush every write immediately —
     // the default in Node, where append is real and cheap; the browser ramps to 15s to avoid rewriting
@@ -132,6 +137,11 @@ export type StorageFactory = (path: string) => Promise<FileStorage>;
 const LOAD_SIGNAL = NULL + "load";
 const OVERLAY_SIGNAL = NULL + "overlay";
 
+// Over network storage we skip automatic (background) compaction by default — reading and rewriting whole
+// files over the network is expensive. An app that wants it anyway opts in once via
+// BulkDatabaseBase.enableNetworkCompaction(). Explicit compact()/tryMergeNow() calls are unaffected.
+let networkCompactionEnabled = false;
+
 let fileNameCounter = 0;
 // Random per-process id baked into file names so two processes (tabs) writing the same collection
 // never collide on a name when they pick the same timestamp/counter in the same millisecond.
@@ -216,7 +226,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // ramping delay (see WRITE_FLUSH_FIRST_STEP_MS / writeFlushMaxDelayMs). Each buffered item keeps an
     // `apply` that re-applies its overlay mutation, so resetReader (which clears the overlay) can restore
     // writes that aren't on disk yet. Removed from the buffer only once their append succeeds.
-    private pendingAppends: { framed: Buffer; apply: () => void }[] = [];
+    private pendingAppends: { framed: Buffer; apply: () => void; rows: number }[] = [];
     private flushTimer: ReturnType<typeof setTimeout> | undefined;
     private flushChain: Promise<void> = Promise.resolve();
     private currentFlushDelay = 0;
@@ -228,8 +238,34 @@ export class BulkDatabaseBase<T extends { key: string }> {
         blockCache.clear();
     }
 
+    // Opt in to automatic compaction even when the storage is remote (off by default — see
+    // networkCompactionEnabled). Global; affects every collection. Explicit compact()/tryMergeNow() always
+    // run regardless of this.
+    public static enableNetworkCompaction() {
+        networkCompactionEnabled = true;
+    }
+
     public storage = lazy(async () => this.storageFactory(`${BULK_ROOT_FOLDER}/${this.name}`));
 
+    // True when this collection's storage is served over the network (a remote server). Apps can branch on
+    // this to adjust behavior for the higher latency (e.g. show a "slower storage" hint, prefetch less).
+    public async isRemote(): Promise<boolean> {
+        return !!(await this.storage()).isRemote;
+    }
+
+    // Whether the tier-0 stream is big enough to fold into bulk now (both bounds): too many rows AND too
+    // many bytes to keep reading whole. See bulkDatabase2Timing.streamFoldTrigger*.
+    private streamNeedsFold(): boolean {
+        return this.streamRowsOnDisk >= bulkDatabase2Timing.streamFoldTriggerRows && this.streamBytesOnDisk > bulkDatabase2Timing.streamFoldTriggerBytes;
+    }
+
+    // Automatic (background) compaction is skipped over the network unless the app opted in. Explicit
+    // compact()/tryMergeNow() bypass this.
+    private async automaticCompactionAllowed(): Promise<boolean> {
+        if (networkCompactionEnabled) return true;
+        return !(await this.storage()).isRemote;
+    }
+
     // In-memory overlay of pending writes/deletes. It takes priority over the loaded readers, so writes
     // are reflected in reads without reloading. Reads observe the relevant signal; mutations invalidate it.
     //
@@ -242,6 +278,23 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // overlay to decide whether an incoming remote write is actually newer than what we have.
     private streamTimes = new Map<string, number>();
 
+    // Cache of fully-resolved (overlay-patched) column results, keyed by column name. The result only
+    // changes when the overlay mutates or the reader resets, so we keep it until then — repeat whole-column
+    // reads (common in a UI re-render) are then free, however large the column. Each cached array is frozen
+    // SHALLOWLY: Object.freeze locks the array itself (callers can't mutate a shared result) but never the
+    // element objects or their values, so typed-array column values keep their fast representation.
+    private columnCache = new Map<string, { key: string; value: unknown; time: number }[]>();
+    // Bumped on every overlay mutation / reader reset. An async column build captures it before its awaits
+    // and only caches its result if it hasn't changed since — so a write or reset mid-build (which clears
+    // the cache and may swap the reader) can never leave a stale entry behind.
+    private dataGen = 0;
+
+    // Approximate size of the tier-0 stream data on disk: set accurately from the last reader build, then
+    // kept current as each flush appends more. Drives the stream-fold trigger (see streamNeedsFold);
+    // reset on resetReader, since after a merge/reset the next reader build re-measures it.
+    private streamRowsOnDisk = 0;
+    private streamBytesOnDisk = 0;
+
     // This instance's tier-0 stream file. Each instance (≈ each thread/tab) streams to its own file
     // so concurrent writers never touch the same file.
     private streamFileName: string | undefined;
@@ -262,6 +315,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
     }
 
     private invalidateOverlay(key: string) {
+        this.dataGen++;
+        this.columnCache.clear();
         this.deps.invalidate(key);
         this.deps.invalidate(OVERLAY_SIGNAL);
     }
@@ -319,6 +374,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
         if (streamData.missing) filesChanged = true;
         if (filesChanged && !tolerateMissing) throw new FilesChangedError();
 
+        // Accurate stream size as of this load; subsequent flushes keep it current (see doFlush).
+        this.streamRowsOnDisk = streamData.entries.length;
+        this.streamBytesOnDisk = streamData.totalBytes;
+
         const bulkReaders = bulkReadersRaw.filter((r): r is BaseBulkDatabaseReader => !!r);
         // The join resolves purely by write-time, so reader order doesn't matter.
         const readers: BaseBulkDatabaseReader[] = [];
@@ -389,6 +448,12 @@ export class BulkDatabaseBase<T extends { key: string }> {
             this.baseFields.clear();
             this.baseFieldsLoading.clear();
             this.overlay.clear();
+            this.dataGen++;
+            this.columnCache.clear();
+            // The next reader build re-measures the stream; clear the estimate so a just-folded stream
+            // doesn't keep looking "heavy" and re-trigger a fold before then.
+            this.streamRowsOnDisk = 0;
+            this.streamBytesOnDisk = 0;
             for (const p of this.pendingAppends) p.apply();
             this.deps.invalidate(LOAD_SIGNAL);
             this.deps.invalidate(OVERLAY_SIGNAL);
@@ -422,7 +487,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const apply = () => { for (const { time, row } of stamped) this.setOverlayRow(row.key as string, row, time); };
         this.deps.batch(apply);
         for (const { time, row } of stamped) syncBroadcast(this.name, { key: row.key as string, time, value: row });
-        await this.streamAppend(framed, apply);
+        await this.streamAppend(framed, apply, stamped.length);
         void this.maybeMerge();
     }
 
@@ -437,7 +502,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const apply = () => { for (const { time, key } of stamped) this.setOverlayDeleted(key, time); };
         this.deps.batch(apply);
         for (const { time, key } of stamped) syncBroadcast(this.name, { key, time, deleted: true });
-        await this.streamAppend(frameDeletes(stamped), apply);
+        await this.streamAppend(frameDeletes(stamped), apply, stamped.length);
         void this.maybeMerge();
     }
 
@@ -445,8 +510,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // `apply` re-applies this write's overlay mutation if resetReader clears the overlay before it's on
     // disk. Awaits durability only on an immediate (idle/Node) flush, so a single action — then close — is
     // saved at once; a burst returns fast and is flushed in the background.
-    private async streamAppend(framed: Buffer, apply: () => void): Promise<void> {
-        this.pendingAppends.push({ framed, apply });
+    private async streamAppend(framed: Buffer, apply: () => void, rows: number): Promise<void> {
+        this.pendingAppends.push({ framed, apply, rows });
         const max = bulkDatabase2Timing.writeFlushMaxDelayMs;
         const now = Date.now();
         // Immediate when batching is off (Node / real append), or for the first write after a lull.
@@ -487,6 +552,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
         await storage.append(this.getStreamFileName(), combined);
         // New writes added during the await are after `batch`, so removing the front is exactly the flushed set.
         this.pendingAppends.splice(0, batch.length);
+        // Keep the stream-size estimate current so the fold trigger sees write-accumulation between reads.
+        this.streamBytesOnDisk += combined.length;
+        for (const p of batch) this.streamRowsOnDisk += p.rows;
     }
 
     public async update(entry: Partial<T> & { key: string }): Promise<void> {
@@ -603,10 +671,13 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return entries.map(e => e.entry);
     }
 
-    // Throttled, fire-and-forget after writes: run a background merge check at most once per interval.
+    // Throttled, fire-and-forget after writes: run a background merge check at most once per interval — but
+    // a tier-0 stream that's grown too big folds promptly, bypassing the throttle. Skipped entirely over
+    // the network unless the app opted in (see automaticCompactionAllowed).
     private async maybeMerge(): Promise<void> {
+        if (!await this.automaticCompactionAllowed()) return;
         const now = Date.now();
-        if (now - this.lastMergeCheck < bulkDatabase2Timing.mergeCheckIntervalMs) return;
+        if (!this.streamNeedsFold() && now - this.lastMergeCheck < bulkDatabase2Timing.mergeCheckIntervalMs) return;
         this.lastMergeCheck = now;
         try {
             await this.tryMergeNow();
@@ -951,8 +1022,12 @@ export class BulkDatabaseBase<T extends { key: string }> {
                 if (recentBytes >= FIRST_MERGE_BYTES) break;
             }
             const span = recent.length ? recent[0].time - recent[recent.length - 1].time : 0;
-            const triggered = recent.length >= 2
-                && (recent.length > bulkDatabase2Timing.firstMergeTriggerFiles || span > bulkDatabase2Timing.firstMergeTriggerRangeMs);
+            // Fold when there's enough fragmentation to consolidate, OR when a foldable stream has grown too
+            // big to keep reading whole (even if it's the only recent file) — see streamNeedsFold.
+            const heavyStreamRecent = this.streamNeedsFold() && recent.some(i => i.kind === "stream");
+            const triggered =
+                recent.length >= 2 && (recent.length > bulkDatabase2Timing.firstMergeTriggerFiles || span > bulkDatabase2Timing.firstMergeTriggerRangeMs)
+                || heavyStreamRecent && recent.length >= 1;
             if (triggered) {
                 const rb = recent.filter(i => i.kind === "bulk").map(i => (i.file as BulkFileInfo));
                 const rs = recent.filter(i => i.kind === "stream").map(i => (i.file as StreamFileInfo));
@@ -1074,14 +1149,21 @@ export class BulkDatabaseBase<T extends { key: string }> {
 
     public async getColumn<Column extends keyof T>(column: Column): Promise<{ key: string; value: T[Column]; time: number }[]> {
         void this.syncSetup();
+        const col = String(column);
+        const cached = this.columnCache.get(col);
+        if (cached) return cached as { key: string; value: T[Column]; time: number }[];
+        const gen = this.dataGen;
         let time = Date.now();
         let reader = await this.reader();
-        let base = await reader.getColumn(String(column));
-        let result = this.patchColumn(base, String(column)) as { key: string; value: T[Column]; time: number }[];
+        let base = await reader.getColumn(col);
+        let result = this.patchColumn(base, col) as { key: string; value: T[Column]; time: number }[];
         time = Date.now() - time;
         if (time > 50) {
             console.log(`${blue(`${this.name}.getColumn(${JSON.stringify(column)})`)} took ${red(formatTime(time))} ${this.formatInfo(reader)}`);
         }
+        Object.freeze(result);
+        // Only cache if no write/reset happened during the awaits above (else this result may be stale).
+        if (this.dataGen === gen) this.columnCache.set(col, result);
         return result;
     }
 
@@ -1176,12 +1258,19 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // Observe the overlay-wide signal so we recompute once the base arrives or the overlay changes.
         this.deps.observe(OVERLAY_SIGNAL);
         let col = String(column);
+        const cached = this.columnCache.get(col);
+        if (cached) return cached as { key: string; value: T[Column]; time: number }[];
         let base = this.baseColumns.get(col);
         if (!base) {
             this.ensureBaseColumn(col);
             return undefined;
         }
-        return this.patchColumn(base, col) as { key: string; value: T[Column]; time: number }[];
+        // Synchronous (no awaits) → reads the current overlay and caches it atomically; an observer re-runs
+        // (and the cache is cleared) on any later change, so the frozen result is safe to share.
+        let result = this.patchColumn(base, col) as { key: string; value: T[Column]; time: number }[];
+        Object.freeze(result);
+        this.columnCache.set(col, result);
+        return result;
     }
 
     public async getColumnInfo() {

package/storage/FileFolderAPI.d.ts

@@ -51,6 +51,7 @@ export type DirectoryWrapper = {
     readonly kind: "directory";
     readonly name: string;
     readonly fullPath?: string;
+    readonly isRemote?: boolean;
     removeEntry(key: string, options?: {
         recursive?: boolean;
     }): Promise<void>;
@@ -138,6 +139,7 @@ export type NestedFileStorage = {
 };
 export type FileStorage = IStorageRaw & {
     folder: NestedFileStorage;
+    isRemote?: boolean;
 };
 export declare function wrapHandle(handle: DirectoryWrapper): FileStorage;
 export declare function getRemoteFileStorageFactory(url: string, password: string, options?: RemoteOptions): (pathStr: string) => Promise<FileStorage>;

package/storage/FileFolderAPI.tsx

@@ -63,6 +63,9 @@ export type DirectoryWrapper = {
     // Full path from the storage root, for diagnostics/logging (the native handle doesn't expose paths,
     // so it's optional). e.g. "bulkDatabases2/myCollection".
     readonly fullPath?: string;
+    // True when this storage is served over the network (a remote server) rather than a local disk, so
+    // callers can adapt for the higher latency/cost (local/native handles leave it undefined = false).
+    readonly isRemote?: boolean;
     removeEntry(key: string, options?: { recursive?: boolean }): Promise<void>;
     getFileHandle(key: string, options?: { create?: boolean }): Promise<FileWrapper>;
     getDirectoryHandle(key: string, options?: { create?: boolean }): Promise<DirectoryWrapper>;
@@ -509,6 +512,9 @@ export type NestedFileStorage = {
 
 export type FileStorage = IStorageRaw & {
     folder: NestedFileStorage;
+    // Mirrors DirectoryWrapper.isRemote: true when this storage is served over the network. Lets callers
+    // (e.g. BulkDatabase2, which skips automatic compaction over the network by default) adapt.
+    isRemote?: boolean;
 };
 
 let appendQueue = cache((key: string) => {
@@ -705,6 +711,7 @@ export function wrapHandle(handle: DirectoryWrapper): FileStorage {
     return {
         ...wrapHandleFiles(handle),
         folder: wrapHandleNested(handle),
+        isRemote: handle.isRemote,
     };
 }
 

package/storage/remoteFileServer.ts

@@ -317,7 +317,16 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
             if (!accessLog.size) return;
             const rows = [...accessLog.values()].sort((a, b) => b.bytes - a.bytes);
             accessLog.clear();
-            for (const e of rows) console.log(`  [${e.op}] ${e.ip}  ${e.path}  ${e.count}x  ${formatBytes(e.bytes)}`);
+            // Fixed-width columns first so lines align, then the variable-length path last. count is the
+            // number of requests folded into this row (R = requests, not a multiplier).
+            const time = new Date().toTimeString().slice(0, 8);
+            for (const e of rows) {
+                const size = formatBytes(e.bytes).padStart(9);
+                const reqs = (e.count + "R").padStart(6);
+                const op = `[${e.op.padEnd(5)}]`;
+                const ip = e.ip.padEnd(15);
+                console.log(`  ${time}  ${size}  ${reqs}  ${op}  ${ip}  ${e.path}`);
+            }
         }, 5000);
         flushTimer.unref?.();
     }

package/storage/remoteFileStorage.ts

@@ -377,6 +377,7 @@ class RemoteDirectoryWrapper implements DirectoryWrapper {
     // Mirror the native FileSystemDirectoryHandle shape (name/kind/entries) so code written against the
     // native API — e.g. recursive walks using `handle.entries()` — works the same over the network.
     readonly kind = "directory" as const;
+    readonly isRemote = true;
     get name() { return baseName(this.dirPath); }
     get fullPath() { return this.dirPath; }
     async removeEntry(key: string): Promise<void> {