sliftutils 1.7.3 → 1.7.5

package/index.d.ts

@@ -1717,6 +1717,7 @@ declare module "sliftutils/storage/DiskCollection" {
             writeDelay?: number | undefined;
             cbor?: boolean | undefined;
             noPrompt?: boolean | undefined;
+            resyncFromDisk?: boolean | undefined;
             freeze?: "deep" | "shallow" | undefined;
             beforeWrite?: ((update: {
                 newValue: T;
@@ -2177,12 +2178,13 @@ declare module "sliftutils/storage/TransactionStorage" {
         private rawStorage;
         private debugName;
         private writeDelay;
+        private resyncFromDisk;
         cache: Map<string, TransactionEntry>;
         private diskFiles;
         private currentChunk;
         private entryCount;
         private static allStorage;
-        constructor(rawStorage: IStorageRaw, debugName: string, writeDelay?: number);
+        constructor(rawStorage: IStorageRaw, debugName: string, writeDelay?: number, resyncFromDisk?: boolean);
         static compressAll(): Promise<void>;
         private resyncCallbacks;
         watchResync(callback: () => void): void;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.7.3",
+    "version": "1.7.5",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/blockCache.ts

@@ -150,7 +150,11 @@ export class BlockCache {
                 let lastMeta = index.blocks[runEnd - 1];
                 let compEnd = index.offsets[runEnd - 1] + lastMeta.len;
                 let runCompressed = rawGetRange(compStart, compEnd);
-                // Register a per-block promise BEFORE awaiting so concurrent reads dedupe onto it.
+                // Hold this run's promises locally. Registering them in the LRU can evict earlier
+                // blocks of the same run (or of a concurrent run) before we get to await them, so
+                // reading them back out of the cache would silently drop blocks and return a short
+                // buffer. We still register in the LRU so concurrent reads dedupe onto them.
+                let runPromises: Promise<Buffer>[] = [];
                 for (let i = runStart; i < runEnd; i++) {
                     let meta = index.blocks[i];
                     let relOffset = index.offsets[i] - compStart;
@@ -159,17 +163,20 @@ export class BlockCache {
                         if (meta.c) return LZ4.decompress(stored);
                         return stored;
                     });
+                    runPromises.push(blockPromise);
                     this.touch(`${fileId}:${i}`, blockPromise);
                 }
-                for (let i = runStart; i < runEnd; i++) {
-                    let promise = this.blocks.get(`${fileId}:${i}`);
-                    if (promise) parts.push(await promise);
-                }
+                for (let promise of runPromises) parts.push(await promise);
                 block = runEnd;
             }
 
             let combined = parts.length === 1 && parts[0] || Buffer.concat(parts);
             let sliceStart = start - firstBlock * blockSize;
+            // A short combined buffer means we lost blocks; subarray would clamp and hand back
+            // truncated data that every caller treats as valid. Fail loudly instead.
+            if (combined.length < sliceStart + (end - start)) {
+                throw new Error(`Expected ${sliceStart + (end - start)} bytes of blocks for range [${start}, ${end}) of ${fileId}, was ${combined.length}`);
+            }
             return combined.subarray(sliceStart, sliceStart + (end - start));
         };
     }

package/storage/DiskCollection.d.ts

@@ -12,6 +12,7 @@ export declare class DiskCollection<T> implements IStorageSync<T> {
         writeDelay?: number | undefined;
         cbor?: boolean | undefined;
         noPrompt?: boolean | undefined;
+        resyncFromDisk?: boolean | undefined;
         freeze?: "deep" | "shallow" | undefined;
         beforeWrite?: ((update: {
             newValue: T;

package/storage/DiskCollection.ts

@@ -34,6 +34,8 @@ export class DiskCollection<T> implements IStorageSync<T> {
             writeDelay?: number;
             cbor?: boolean;
             noPrompt?: boolean;
+            // Poll disk and reload when another process/tab writes to this collection. Off by default.
+            resyncFromDisk?: boolean;
             freeze?: "shallow" | "deep";
             // May mutate newValue in order to change what will be written
             beforeWrite?: (update: { newValue: T; key: string; collection: DiskCollection<T> }) => void;
@@ -52,7 +54,7 @@ export class DiskCollection<T> implements IStorageSync<T> {
             let collections = await fileStorage.folder.getStorage("collections");
             curCollection = await collections.folder.getStorage(this.collectionName);
         }
-        let baseStorage = new TransactionStorage(curCollection, this.collectionName, this.config?.writeDelay);
+        let baseStorage = new TransactionStorage(curCollection, this.collectionName, this.config?.writeDelay, this.config?.resyncFromDisk);
         this.transactionStorage = baseStorage;
         return this.config?.cbor ? new CBORStorage<T>(baseStorage) : new JSONStorage<T>(baseStorage);
     }

package/storage/TransactionStorage.d.ts

@@ -11,12 +11,13 @@ export declare class TransactionStorage implements IStorage<Buffer> {
     private rawStorage;
     private debugName;
     private writeDelay;
+    private resyncFromDisk;
     cache: Map<string, TransactionEntry>;
     private diskFiles;
     private currentChunk;
     private entryCount;
     private static allStorage;
-    constructor(rawStorage: IStorageRaw, debugName: string, writeDelay?: number);
+    constructor(rawStorage: IStorageRaw, debugName: string, writeDelay?: number, resyncFromDisk?: boolean);
     static compressAll(): Promise<void>;
     private resyncCallbacks;
     watchResync(callback: () => void): void;

package/storage/TransactionStorage.ts

@@ -92,7 +92,11 @@ export class TransactionStorage implements IStorage<Buffer> {
     constructor(
         private rawStorage: IStorageRaw,
         private debugName: string,
-        private writeDelay = WRITE_DELAY
+        private writeDelay = WRITE_DELAY,
+        // When set, we poll the underlying storage and reload if another process/tab writes new
+        // chunk files. Off by default, as it costs a periodic disk scan and most collections are
+        // only written by a single process.
+        private resyncFromDisk = false
     ) {
         TransactionStorage.allStorage.push(this);
         // VERY useful for debugging.
@@ -282,7 +286,7 @@ export class TransactionStorage implements IStorage<Buffer> {
     private async loadAllTransactions(initialLoad?: boolean): Promise<string[]> {
         if (isInBuild()) return [];
 
-        if (initialLoad) {
+        if (initialLoad && this.resyncFromDisk) {
             runInfinitePoll(DISK_CHECK_INTERVAL, () => this.checkDisk());
         }