sliftutils 1.7.2 → 1.7.4

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.7.2",
+    "version": "1.7.4",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/LoadedIndex.ts

@@ -5,6 +5,9 @@ import { STREAM_EXTENSION, StreamEntry, parseStream, streamReaderFromEntries } f
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { blue, red } from "socket-function/src/formatting/logColors";
 
+// Duplicated from BulkDatabaseBase to avoid a circular import — used only to build a fuller storage-relative path for log messages.
+const BULK_ROOT_FOLDER = "bulkDatabases2";
+
 export type BulkFileInfo = { fileName: string; level: number; timestamp: number };
 export type StreamFileInfo = { fileName: string; timestamp: number };
 export type StreamReaderCacheEntry = { readSize: number; parsedPos: number; entries: StreamEntry[] };
@@ -257,7 +260,7 @@ export async function loadFileReader(name: string, storage: FileStorage, f: Bulk
     const raw = await makeRawGetRange(storage, f.fileName);
     const fileId = nullJoin(name, f.fileName);
     const opened = await blockCache.open(fileId, raw.size, raw.rawGetRange);
-    const reader = await loadBulkDatabase({ totalBytes: opened.uncompressedSize, getRange: opened.getRange, name: f.fileName });
+    const reader = await loadBulkDatabase({ totalBytes: opened.uncompressedSize, getRange: opened.getRange, name: `${BULK_ROOT_FOLDER}/${name}/${f.fileName}` });
     cache.set(f.fileName, reader);
     return reader;
 }
@@ -333,11 +336,11 @@ function pruneSubCaches(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[]
 // visible without flooding the log on every rebuild.
 const warnedCorruptReads = new Set<string>();
 function warnCorruptRead(db: BaseBulkDatabaseReader, column: string, e: unknown): void {
-    const fileName = db.name || "(unknown source)";
-    const dedupeKey = nullJoin(fileName, column);
+    const path = db.name || "(unknown source)";
+    const dedupeKey = nullJoin(path, column);
     if (warnedCorruptReads.has(dedupeKey)) return;
     warnedCorruptReads.add(dedupeKey);
-    console.warn(`${red("corrupt read")}: file ${fileName} column ${JSON.stringify(column)} could not be read - skipping it (its data is dropped from results): ${(e as Error).message}`);
+    console.warn(`${red("corrupt read")}: file ${path} column ${JSON.stringify(column)} could not be read - skipping it (its data is dropped from results): ${(e as Error).message}`);
 }
 
 function joinBulkDatabases(databases: BaseBulkDatabaseReader[]): ResolvedReader {

package/storage/BulkDatabase2/blockCache.ts

@@ -150,7 +150,11 @@ export class BlockCache {
                 let lastMeta = index.blocks[runEnd - 1];
                 let compEnd = index.offsets[runEnd - 1] + lastMeta.len;
                 let runCompressed = rawGetRange(compStart, compEnd);
-                // Register a per-block promise BEFORE awaiting so concurrent reads dedupe onto it.
+                // Hold this run's promises locally. Registering them in the LRU can evict earlier
+                // blocks of the same run (or of a concurrent run) before we get to await them, so
+                // reading them back out of the cache would silently drop blocks and return a short
+                // buffer. We still register in the LRU so concurrent reads dedupe onto them.
+                let runPromises: Promise<Buffer>[] = [];
                 for (let i = runStart; i < runEnd; i++) {
                     let meta = index.blocks[i];
                     let relOffset = index.offsets[i] - compStart;
@@ -159,17 +163,20 @@ export class BlockCache {
                         if (meta.c) return LZ4.decompress(stored);
                         return stored;
                     });
+                    runPromises.push(blockPromise);
                     this.touch(`${fileId}:${i}`, blockPromise);
                 }
-                for (let i = runStart; i < runEnd; i++) {
-                    let promise = this.blocks.get(`${fileId}:${i}`);
-                    if (promise) parts.push(await promise);
-                }
+                for (let promise of runPromises) parts.push(await promise);
                 block = runEnd;
             }
 
             let combined = parts.length === 1 && parts[0] || Buffer.concat(parts);
             let sliceStart = start - firstBlock * blockSize;
+            // A short combined buffer means we lost blocks; subarray would clamp and hand back
+            // truncated data that every caller treats as valid. Fail loudly instead.
+            if (combined.length < sliceStart + (end - start)) {
+                throw new Error(`Expected ${sliceStart + (end - start)} bytes of blocks for range [${start}, ${end}) of ${fileId}, was ${combined.length}`);
+            }
             return combined.subarray(sliceStart, sliceStart + (end - start));
         };
     }