sliftutils 1.2.39 → 1.3.1

package/storage/BulkDatabase2/BulkDatabaseFormat.d.ts

@@ -0,0 +1,32 @@
+/// <reference types="node" />
+/// <reference types="node" />
+export declare const KEY_COLUMN = "key";
+export declare const EMPTY_BUFFER: Buffer;
+export declare const ABSENT: unique symbol;
+export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[]): Buffer[];
+export type BaseBulkDatabaseReader = {
+    rowCount: number;
+    totalBytes: number;
+    minTime: number;
+    maxTime: number;
+    keys: string[];
+    columns: {
+        column: string;
+        byteSize: number;
+    }[];
+    keyTimes: Map<string, number>;
+    deleteTimes?: Map<string, number>;
+    getColumn: (column: string) => Promise<{
+        key: string;
+        value: unknown;
+        time: number;
+    }[]>;
+    getSingleField: (key: string, column: string) => Promise<{
+        value: unknown;
+        time: number;
+    } | typeof ABSENT>;
+};
+export declare function loadBulkDatabase(config: {
+    totalBytes: number;
+    getRange: (start: number, end: number) => Promise<Buffer>;
+}): Promise<BaseBulkDatabaseReader>;

package/storage/BulkDatabase2/BulkDatabaseFormat.ts

@@ -0,0 +1,331 @@
+// File format (all integers little-endian):
+//   u32 headerLength, then headerLength bytes of JSON: { rowCount, columns: [{ name, offset, length }] }, where offset is relative to the end of the header.
+//   Each column blob is: u32 offsets[rowCount + 1] (byte offsets into the column's data section), u8 types[rowCount], then the data section (each value's bytes, concatenated).
+// Values are encoded with an explicit type tag per value (see the TYPE_ constants). Reads only fetch the byte ranges they need.
+
+export const KEY_COLUMN = "key";
+
+// Every tag is an explicit constant baked into files on disk. NEVER renumber an existing tag — only add new ones — or every previously written file silently decodes wrong.
+const TYPE_UNDEFINED = 0;
+const TYPE_STRING = 1;
+const TYPE_NUMBER = 2;
+const TYPE_BOOLEAN = 3;
+const TYPE_OBJECT = 4;
+const TYPE_INT8_ARRAY = 5;
+const TYPE_UINT8_ARRAY = 6;
+const TYPE_UINT8_CLAMPED_ARRAY = 7;
+const TYPE_INT16_ARRAY = 8;
+const TYPE_UINT16_ARRAY = 9;
+const TYPE_INT32_ARRAY = 10;
+const TYPE_UINT32_ARRAY = 11;
+const TYPE_FLOAT32_ARRAY = 12;
+const TYPE_FLOAT64_ARRAY = 13;
+// A cell whose row never set this column at all — as opposed to TYPE_UNDEFINED, an explicitly stored
+// undefined. On read, ABSENT falls through to older readers for that column; a stored undefined stops
+// the fall-through (it's a real value that clears the column).
+const TYPE_ABSENT = 14;
+
+const TYPED_ARRAY_TYPES: { type: number; ctor: { new(buffer: ArrayBuffer): ArrayBufferView; BYTES_PER_ELEMENT: number; name: string } }[] = [
+    { type: TYPE_INT8_ARRAY, ctor: Int8Array },
+    { type: TYPE_UINT8_ARRAY, ctor: Uint8Array },
+    { type: TYPE_UINT8_CLAMPED_ARRAY, ctor: Uint8ClampedArray },
+    { type: TYPE_INT16_ARRAY, ctor: Int16Array },
+    { type: TYPE_UINT16_ARRAY, ctor: Uint16Array },
+    { type: TYPE_INT32_ARRAY, ctor: Int32Array },
+    { type: TYPE_UINT32_ARRAY, ctor: Uint32Array },
+    { type: TYPE_FLOAT32_ARRAY, ctor: Float32Array },
+    { type: TYPE_FLOAT64_ARRAY, ctor: Float64Array },
+];
+
+export const EMPTY_BUFFER = Buffer.alloc(0) as Buffer;
+
+// Sentinel a reader returns for a cell whose row never set this column, so the join can fall through
+// to an older reader for that column. Distinct from a stored undefined, which is a real clearing value.
+export const ABSENT = Symbol("absent");
+
+// Hidden per-row column holding each row's write-time (so reads can resolve a key to its latest value
+// by actual time). NUL-prefixed so it can't collide with a user column; excluded from `columns`.
+const TIME_COLUMN = String.fromCharCode(0) + "t";
+
+type FileHeader = {
+    rowCount: number;
+    columns: { name: string; offset: number; length: number }[];
+    // Oldest/newest write-time of the data in this file (from the stream entries it was folded from,
+    // carried through merges). Lets the reader order bulk files by actual data recency and lets merges
+    // assert that two files' time ranges never overlap. Absent (0) in files written before this existed.
+    minTime?: number;
+    maxTime?: number;
+};
+
+function encodeValue(value: unknown): { type: number; bytes: Buffer } {
+    if (value === ABSENT) {
+        return { type: TYPE_ABSENT, bytes: EMPTY_BUFFER };
+    }
+    if (value === undefined || value === null) {
+        return { type: TYPE_UNDEFINED, bytes: EMPTY_BUFFER };
+    }
+    if (typeof value === "string") {
+        return { type: TYPE_STRING, bytes: Buffer.from(value, "utf8") };
+    }
+    if (typeof value === "number") {
+        const bytes = Buffer.alloc(8);
+        bytes.writeDoubleLE(value, 0);
+        return { type: TYPE_NUMBER, bytes };
+    }
+    if (typeof value === "boolean") {
+        return { type: TYPE_BOOLEAN, bytes: Buffer.from([value && 1 || 0]) };
+    }
+    if (ArrayBuffer.isView(value)) {
+        if (value instanceof DataView) {
+            throw new Error(`DataView values are not supported, store a typed array instead`);
+        }
+        const entry = TYPED_ARRAY_TYPES.find(t => value instanceof t.ctor);
+        if (!entry) {
+            throw new Error(`Unsupported typed array type ${value.constructor.name}`);
+        }
+        return {
+            type: entry.type,
+            bytes: Buffer.from(value.buffer, value.byteOffset, value.byteLength),
+        };
+    }
+    if (typeof value !== "object") {
+        throw new Error(`Unsupported value type ${typeof value}`);
+    }
+    return { type: TYPE_OBJECT, bytes: Buffer.from(JSON.stringify(value), "utf8") };
+}
+
+function decodeValue(type: number, bytes: Buffer): unknown {
+    if (type === TYPE_UNDEFINED) return undefined;
+    if (type === TYPE_STRING) return bytes.toString("utf8");
+    if (type === TYPE_NUMBER) {
+        if (bytes.length !== 8) {
+            throw new Error(`Expected 8 bytes for a number, was ${bytes.length}`);
+        }
+        return bytes.readDoubleLE(0);
+    }
+    if (type === TYPE_BOOLEAN) {
+        if (bytes.length !== 1) {
+            throw new Error(`Expected 1 byte for a boolean, was ${bytes.length}`);
+        }
+        return bytes[0] === 1;
+    }
+    if (type === TYPE_OBJECT) return JSON.parse(bytes.toString("utf8"));
+    if (type === TYPE_ABSENT) return ABSENT;
+    const entry = TYPED_ARRAY_TYPES.find(t => t.type === type);
+    if (!entry) {
+        throw new Error(`Expected a valid type tag, was ${type}`);
+    }
+    const ctor = entry.ctor;
+    if (type === TYPE_UINT8_ARRAY) return Buffer.from(bytes);
+    if (bytes.length % ctor.BYTES_PER_ELEMENT !== 0) {
+        throw new Error(`Expected byte length divisible by ${ctor.BYTES_PER_ELEMENT} for ${ctor.name}, was ${bytes.length}`);
+    }
+    // Copy to a fresh ArrayBuffer so the typed array view is aligned regardless of where the bytes landed in the source buffer.
+    const aligned = new ArrayBuffer(bytes.length);
+    new Uint8Array(aligned).set(bytes);
+    return new ctor(aligned);
+}
+
+function encodeBulkData(data: unknown[]): Buffer {
+    const n = data.length;
+    const offsets = Buffer.alloc(4 * (n + 1));
+    const types = Buffer.alloc(n);
+    const parts: Buffer[] = [];
+    let pos = 0;
+    for (let i = 0; i < n; i++) {
+        const { type, bytes } = encodeValue(data[i]);
+        offsets.writeUInt32LE(pos, 4 * i);
+        types[i] = type;
+        parts.push(bytes);
+        pos += bytes.length;
+    }
+    offsets.writeUInt32LE(pos, 4 * n);
+    return Buffer.concat([offsets, types, ...parts]);
+}
+
+function decodeBulkData(blob: Buffer, rowCount: number): unknown[] {
+    const indexSize = 4 * (rowCount + 1) + rowCount;
+    const values: unknown[] = [];
+    for (let i = 0; i < rowCount; i++) {
+        const start = blob.readUInt32LE(4 * i);
+        const end = blob.readUInt32LE(4 * (i + 1));
+        const type = blob[4 * (rowCount + 1) + i];
+        values.push(decodeValue(type, blob.subarray(indexSize + start, indexSize + end)));
+    }
+    return values;
+}
+
+// Past this estimated logical size a single rows array is split across multiple files, so no one
+// file (and therefore no single column blob, and no single Buffer.concat) ever approaches the
+// ~2GB Buffer length limit. The cap is deliberately the same as the merge cap: 800MB is the most
+// we'll ever hold for one file. The estimate is rough on purpose — it only needs to keep each
+// chunk comfortably under the limit, not be exact.
+const FILE_SPLIT_BYTES = 800 * 1024 * 1024;
+
+// Per-value on-disk overhead: a 4-byte offset entry plus a 1-byte type tag.
+const PER_VALUE_OVERHEAD = 5;
+
+function estimateValueBytes(value: unknown): number {
+    if (value === undefined || value === null) return 0;
+    if (typeof value === "string") return value.length * 2;
+    if (typeof value === "number") return 8;
+    if (typeof value === "boolean") return 1;
+    if (ArrayBuffer.isView(value)) return value.byteLength;
+    if (typeof value === "object") return JSON.stringify(value).length;
+    return 0;
+}
+
+function estimateRowBytes(row: Record<string, unknown>): number {
+    let total = 0;
+    for (const value of Object.values(row)) {
+        total += estimateValueBytes(value) + PER_VALUE_OVERHEAD;
+    }
+    return total;
+}
+
+function buildOneFile(rows: Record<string, unknown>[], times: number[]): Buffer {
+    const columnNames: string[] = [];
+    const columnSet = new Set<string>();
+    for (const row of rows) {
+        for (const field of Object.keys(row)) {
+            if (columnSet.has(field)) continue;
+            columnSet.add(field);
+            columnNames.push(field);
+        }
+    }
+    // A row that doesn't include a column stores ABSENT (fall-through), not undefined (a real value).
+    const blobs = columnNames.map(col => encodeBulkData(rows.map(row => col in row ? row[col] : ABSENT)));
+    // The hidden per-row time column.
+    columnNames.push(TIME_COLUMN);
+    blobs.push(encodeBulkData(times));
+    let offset = 0;
+    const columns = columnNames.map((name, i) => {
+        const entry = { name, offset, length: blobs[i].length };
+        offset += blobs[i].length;
+        return entry;
+    });
+    let minTime = times.length ? times[0] : 0;
+    let maxTime = minTime;
+    for (const t of times) { if (t < minTime) minTime = t; if (t > maxTime) maxTime = t; }
+    const header: FileHeader = { rowCount: rows.length, columns, minTime, maxTime };
+    const headerBuf = Buffer.from(JSON.stringify(header), "utf8");
+    const lengthPrefix = Buffer.alloc(4);
+    lengthPrefix.writeUInt32LE(headerBuf.length, 0);
+    return Buffer.concat([lengthPrefix, headerBuf, ...blobs]);
+}
+
+// Returns one complete, independent file buffer per chunk of rows. When the caller hands us more
+// rows than fit comfortably in one file we partition by row range — each returned buffer is exactly
+// what buildOneFile would produce if called with that subset, so the chunks have disjoint keys and
+// the caller just writes each as its own file. A normal-sized write returns a single buffer.
+// `times[i]` is row i's write-time, stored per row so reads resolve a key to its latest value by time.
+export function buildFileBuffer(rows: Record<string, unknown>[], times: number[]): Buffer[] {
+    if (rows.length === 0) return [buildOneFile([], [])];
+    const result: Buffer[] = [];
+    let chunkStart = 0;
+    let chunkBytes = 0;
+    for (let i = 0; i < rows.length; i++) {
+        const rowBytes = estimateRowBytes(rows[i]);
+        if (i > chunkStart && chunkBytes + rowBytes > FILE_SPLIT_BYTES) {
+            result.push(buildOneFile(rows.slice(chunkStart, i), times.slice(chunkStart, i)));
+            chunkStart = i;
+            chunkBytes = 0;
+        }
+        chunkBytes += rowBytes;
+    }
+    result.push(buildOneFile(rows.slice(chunkStart), times.slice(chunkStart)));
+    return result;
+}
+
+export type BaseBulkDatabaseReader = {
+    rowCount: number;
+    totalBytes: number;
+    // Write-time bounds of this reader's data (0 if unknown — old bulk files). Diagnostics only now.
+    minTime: number;
+    maxTime: number;
+    // Keys is special, it's always automatically decoded, even though it is stored as a normal column
+    keys: string[];
+    columns: { column: string; byteSize: number }[];
+    // Each key's row write-time (the time of its newest write in this reader). The join compares these
+    // across readers to resolve a key to its latest value.
+    keyTimes: Map<string, number>;
+    // Per-key tombstone time: the key was deleted at this time. The join treats a delete like any other
+    // event — a delete only wins if it's newer than every set for the key. Only the stream reader sets it.
+    deleteTimes?: Map<string, number>;
+    // Each key's value for the column plus the row's write-time. value may be ABSENT (the row never set
+    // this column — the join then falls through to an older reader for that key/column).
+    getColumn: (column: string) => Promise<{ key: string; value: unknown; time: number }[]>;
+    // The value + write-time for (key, column), or ABSENT if this reader has no such cell.
+    getSingleField: (key: string, column: string) => Promise<{ value: unknown; time: number } | typeof ABSENT>;
+};
+
+export async function loadBulkDatabase(config: {
+    totalBytes: number;
+    getRange: (start: number, end: number) => Promise<Buffer>;
+}): Promise<BaseBulkDatabaseReader> {
+    const headerLength = (await config.getRange(0, 4)).readUInt32LE(0);
+    if (headerLength <= 0 || headerLength > config.totalBytes) {
+        throw new Error(`Expected header length in (0, ${config.totalBytes}], was ${headerLength}`);
+    }
+    const header = JSON.parse((await config.getRange(4, 4 + headerLength)).toString("utf8")) as FileHeader;
+    const dataBase = 4 + headerLength;
+    const rowCount = header.rowCount;
+    const colByName = new Map(header.columns.map(c => [c.name, c]));
+
+    async function readWholeColumn(column: string): Promise<unknown[]> {
+        const col = colByName.get(column);
+        if (!col) {
+            throw new Error(`Expected column ${column}, file only has: ${header.columns.map(c => c.name).join(", ")}`);
+        }
+        const blob = await config.getRange(dataBase + col.offset, dataBase + col.offset + col.length);
+        return decodeBulkData(blob, rowCount);
+    }
+
+    const keys = (await readWholeColumn(KEY_COLUMN)).map(v => {
+        if (typeof v !== "string") {
+            throw new Error(`Expected string key, was ${typeof v}: ${JSON.stringify(v)?.slice(0, 500)}`);
+        }
+        return v;
+    });
+    const keyIndex = new Map(keys.map((key, i) => [key, i]));
+
+    // Per-row write-times. Old files (written before this column existed) fall back to the file's header
+    // time (or 0) for every row — fine, since such files predate concurrent-time resolution.
+    const times: number[] = colByName.has(TIME_COLUMN)
+        ? (await readWholeColumn(TIME_COLUMN)).map(v => typeof v === "number" ? v : 0)
+        : keys.map(() => header.maxTime || 0);
+
+    return {
+        rowCount,
+        totalBytes: config.totalBytes,
+        minTime: header.minTime || 0,
+        maxTime: header.maxTime || 0,
+        keys,
+        keyTimes: new Map(keys.map((key, i) => [key, times[i]])),
+        columns: header.columns.filter(c => c.name !== TIME_COLUMN).map(c => ({ column: c.name, byteSize: c.length })),
+        async getColumn(column) {
+            const values = await readWholeColumn(column);
+            return keys.map((key, i) => ({ key, value: values[i], time: times[i] }));
+        },
+        async getSingleField(key, column) {
+            const row = keyIndex.get(key);
+            if (row === undefined) return ABSENT;
+            const col = colByName.get(column);
+            if (!col) return ABSENT;
+            const colBase = dataBase + col.offset;
+            const offsetsBuf = await config.getRange(colBase + 4 * row, colBase + 4 * row + 8);
+            const start = offsetsBuf.readUInt32LE(0);
+            const end = offsetsBuf.readUInt32LE(4);
+            const typePos = colBase + 4 * (rowCount + 1) + row;
+            const typeBuf = await config.getRange(typePos, typePos + 1);
+            const dataStart = colBase + 4 * (rowCount + 1) + rowCount;
+            let bytes = EMPTY_BUFFER;
+            if (end > start) {
+                bytes = await config.getRange(dataStart + start, dataStart + end);
+            }
+            const value = decodeValue(typeBuf[0], bytes);
+            if (value === ABSENT) return ABSENT;
+            return { value, time: times[row] };
+        },
+    };
+}

package/storage/BulkDatabase2/blockCache.d.ts

@@ -0,0 +1,17 @@
+/// <reference types="node" />
+/// <reference types="node" />
+export type GetRange = (start: number, end: number) => Promise<Buffer>;
+export declare function encodeCompressedBlocks(data: Buffer): Buffer;
+export declare class BlockCache {
+    private blocks;
+    private indexes;
+    clear(): void;
+    private touch;
+    private readIndex;
+    open(fileId: string, fileSize: number, rawGetRange: GetRange): Promise<{
+        uncompressedSize: number;
+        getRange: GetRange;
+    }>;
+    private makeGetRange;
+}
+export declare const blockCache: BlockCache;

package/storage/BulkDatabase2/blockCache.ts

@@ -0,0 +1,173 @@
+import { LZ4 } from "socket-function/src/lz4/LZ4";
+
+// Block-aligned, promise-deduped, decompressing range cache for BulkDatabase2's on-disk files.
+//
+// On-disk format (produced by encodeCompressedBlocks): the logical (uncompressed) bulk buffer is
+// split into fixed BLOCK_SIZE blocks; each block is LZ4-compressed unless that wouldn't shrink it by
+// at least 2x, in which case it's stored raw. The file starts with a JSON index mapping each block to
+// its stored length + a compressed flag, so a reader can seek straight to any block:
+//
+//   [u32 indexLength][index JSON][block 0 stored bytes][block 1 stored bytes]...
+//
+// The cache presents a *logical* getRange (over uncompressed bytes) so BulkDatabase2's reader is
+// oblivious to compression. It reads compressed bytes via the underlying getRange, decompresses, and
+// caches the uncompressed blocks. Reads are promise-deduped and contiguous missing blocks are
+// coalesced into one underlying read. Files are immutable, so cached blocks are valid forever.
+//
+// Per-block compression mainly helps slow storage (HDD): fewer bytes off disk per block, at the cost
+// of a fast in-memory LZ4 decompress.
+
+const BLOCK_SIZE = 256 * 1024;
+const MAX_BLOCKS = Math.floor((512 * 1024 * 1024) / BLOCK_SIZE);
+const MIN_COMPRESSION_RATIO = 2;
+
+const EMPTY = Buffer.alloc(0) as Buffer;
+
+export type GetRange = (start: number, end: number) => Promise<Buffer>;
+
+type BlockMeta = { len: number; c: 0 | 1 };
+type FileIndex = {
+    uncompressedSize: number;
+    blockSize: number;
+    blocks: BlockMeta[];
+    // Stored byte offset of each block (offsets[i]..offsets[i]+blocks[i].len), filled in on parse.
+    offsets: number[];
+};
+
+// Writes a logical bulk buffer in the compressed-block format described above.
+export function encodeCompressedBlocks(data: Buffer): Buffer {
+    let blocks: BlockMeta[] = [];
+    let storedParts: Buffer[] = [];
+    let blockCount = Math.ceil(data.length / BLOCK_SIZE);
+    for (let i = 0; i < blockCount; i++) {
+        let raw = data.subarray(i * BLOCK_SIZE, Math.min((i + 1) * BLOCK_SIZE, data.length));
+        let compressed = LZ4.compress(raw);
+        // Only keep the compressed form if it shrinks the block by at least MIN_COMPRESSION_RATIO.
+        if (compressed.length * MIN_COMPRESSION_RATIO <= raw.length) {
+            storedParts.push(compressed);
+            blocks.push({ len: compressed.length, c: 1 });
+        } else {
+            storedParts.push(raw);
+            blocks.push({ len: raw.length, c: 0 });
+        }
+    }
+    let indexBuf = Buffer.from(JSON.stringify({ uncompressedSize: data.length, blockSize: BLOCK_SIZE, blocks }), "utf8");
+    let prefix = Buffer.alloc(4);
+    prefix.writeUInt32LE(indexBuf.length, 0);
+    return Buffer.concat([prefix, indexBuf, ...storedParts]);
+}
+
+export class BlockCache {
+    // Uncompressed blocks; insertion order is LRU order (re-inserted on access).
+    private blocks = new Map<string, Promise<Buffer>>();
+    // Parsed file index per fileId (small; not subject to the block LRU).
+    private indexes = new Map<string, Promise<FileIndex>>();
+
+    public clear() {
+        this.blocks.clear();
+        this.indexes.clear();
+    }
+
+    private touch(key: string, value: Promise<Buffer>) {
+        this.blocks.delete(key);
+        this.blocks.set(key, value);
+        while (this.blocks.size > MAX_BLOCKS) {
+            let oldest = this.blocks.keys().next().value;
+            if (oldest === undefined) break;
+            this.blocks.delete(oldest);
+        }
+    }
+
+    // Reads + validates the file index. Rejects the WHOLE file if it isn't exactly the size its index
+    // implies — a truncated/partial write (e.g. a crash mid-write) is detected here rather than
+    // silently returning corrupted values for the rows near the end.
+    private async readIndex(rawGetRange: GetRange, fileSize: number): Promise<FileIndex> {
+        let head = await rawGetRange(0, 4);
+        if (head.length < 4) throw new Error(`bulk file too short for an index header (${fileSize} bytes)`);
+        let indexLength = head.readUInt32LE(0);
+        if (indexLength <= 0 || 4 + indexLength > fileSize) {
+            throw new Error(`bulk file index length ${indexLength} is invalid for a ${fileSize}-byte file`);
+        }
+        let indexBuf = await rawGetRange(4, 4 + indexLength);
+        let parsed = JSON.parse(indexBuf.toString("utf8")) as { uncompressedSize: number; blockSize: number; blocks: BlockMeta[] };
+        let dataBase = 4 + indexLength;
+        let offsets: number[] = [];
+        let offset = dataBase;
+        for (let block of parsed.blocks) {
+            offsets.push(offset);
+            offset += block.len;
+        }
+        // The blocks must account for exactly the rest of the file. Any mismatch means the file is
+        // truncated or otherwise corrupt — reject it entirely.
+        if (offset !== fileSize) {
+            throw new Error(`bulk file is ${fileSize} bytes but its index implies ${offset} (truncated/corrupt)`);
+        }
+        return { ...parsed, offsets };
+    }
+
+    // Opens an immutable file: reads + validates its index (cached) and returns the logical
+    // (uncompressed) size plus a logical getRange that the caller can use exactly like an
+    // uncompressed file. `fileSize` is the actual on-disk byte length, used to detect truncation.
+    public async open(fileId: string, fileSize: number, rawGetRange: GetRange): Promise<{ uncompressedSize: number; getRange: GetRange }> {
+        let indexPromise = this.indexes.get(fileId);
+        if (!indexPromise) {
+            indexPromise = this.readIndex(rawGetRange, fileSize);
+            this.indexes.set(fileId, indexPromise);
+        }
+        let index = await indexPromise;
+        return { uncompressedSize: index.uncompressedSize, getRange: this.makeGetRange(fileId, index, rawGetRange) };
+    }
+
+    private makeGetRange(fileId: string, index: FileIndex, rawGetRange: GetRange): GetRange {
+        let blockSize = index.blockSize;
+        return async (start, end) => {
+            if (end <= start) return EMPTY;
+            let firstBlock = Math.floor(start / blockSize);
+            let lastBlock = Math.floor((end - 1) / blockSize);
+
+            let parts: Buffer[] = [];
+            let block = firstBlock;
+            while (block <= lastBlock) {
+                let key = `${fileId}:${block}`;
+                let cached = this.blocks.get(key);
+                if (cached) {
+                    this.touch(key, cached);
+                    parts.push(await cached);
+                    block++;
+                    continue;
+                }
+                // Coalesce a run of consecutive missing blocks into one underlying (compressed) read.
+                let runStart = block;
+                let runEnd = block + 1;
+                while (runEnd <= lastBlock && !this.blocks.has(`${fileId}:${runEnd}`)) runEnd++;
+                let compStart = index.offsets[runStart];
+                let lastMeta = index.blocks[runEnd - 1];
+                let compEnd = index.offsets[runEnd - 1] + lastMeta.len;
+                let runCompressed = rawGetRange(compStart, compEnd);
+                // Register a per-block promise BEFORE awaiting so concurrent reads dedupe onto it.
+                for (let i = runStart; i < runEnd; i++) {
+                    let meta = index.blocks[i];
+                    let relOffset = index.offsets[i] - compStart;
+                    let blockPromise = runCompressed.then(buf => {
+                        let stored = buf.subarray(relOffset, relOffset + meta.len);
+                        if (meta.c) return LZ4.decompress(stored);
+                        return stored;
+                    });
+                    this.touch(`${fileId}:${i}`, blockPromise);
+                }
+                for (let i = runStart; i < runEnd; i++) {
+                    let promise = this.blocks.get(`${fileId}:${i}`);
+                    if (promise) parts.push(await promise);
+                }
+                block = runEnd;
+            }
+
+            let combined = parts.length === 1 && parts[0] || Buffer.concat(parts);
+            let sliceStart = start - firstBlock * blockSize;
+            return combined.subarray(sliceStart, sliceStart + (end - start));
+        };
+    }
+}
+
+// Shared across every BulkDatabase2 collection/file (the 512MB budget is global).
+export const blockCache = new BlockCache();

package/storage/BulkDatabase2/manifest.d.ts

@@ -0,0 +1,17 @@
+export declare const MANIFEST_EXTENSION = ".manifest";
+export type Manifest = {
+    startTime: number;
+    validBulkFiles: string[];
+    ignoredStreamFiles: string[];
+    readFiles: string[];
+};
+export declare function isManifestName(name: string): boolean;
+export declare function manifestFileName(startTime: number, writerId: string, counter: number): string;
+export declare function parseManifestStartTime(name: string): number | undefined;
+export declare function chooseManifest(manifests: {
+    name: string;
+    manifest: Manifest;
+}[]): {
+    name: string;
+    manifest: Manifest;
+} | undefined;

package/storage/BulkDatabase2/manifest.ts

@@ -0,0 +1,59 @@
+// A manifest records which bulk files are valid, decoupling "a file exists on disk" from "a file is
+// part of the database." Every operation that changes the bulk layout (rollover, merge, direct write)
+// writes a brand-new manifest instead of mutating/deleting in place, so changes are atomic from a
+// reader's point of view: a reader sees either the old manifest or the new one, never a half-applied
+// state. Manifests are immutable once written and never clobbered.
+//
+// Resolution: read every manifest, pick the one with the newest startTime (the time its writer
+// snapshotted the directory). The latest starter has the most up-to-date view, so its decision wins;
+// an older-starting writer that finishes later is ignored, and its freshly-written files are simply
+// orphaned (cleaned up later) — its inputs were never marked consumed, so no data is lost.
+//
+// Back-compat: if there is no manifest at all, every bulk file is valid (old databases just work).
+// Stream files are always valid unless a manifest lists them as already merged into a bulk file
+// (ignoredStreamFiles); they carry their own per-write timestamps, so they self-resolve regardless.
+
+export const MANIFEST_EXTENSION = ".manifest";
+
+export type Manifest = {
+    // When the writer snapshotted the directory (its read time). Newest startTime wins.
+    startTime: number;
+    // The full set of bulk files that are valid as of this manifest (not a delta).
+    validBulkFiles: string[];
+    // Stream files already folded into a bulk file — ignore them on read; cleanup deletes them later.
+    ignoredStreamFiles: string[];
+    // The filenames the writer saw at startTime (diagnostics + lets cleanup reason about what existed).
+    readFiles: string[];
+};
+
+export function isManifestName(name: string): boolean {
+    return name.endsWith(MANIFEST_EXTENSION);
+}
+
+// manifest_<startTime>_<writerId>_<counter>.manifest — writerId keeps two processes from colliding on
+// a name when they start in the same millisecond.
+export function manifestFileName(startTime: number, writerId: string, counter: number): string {
+    return `manifest_${startTime}_${writerId}_${counter}${MANIFEST_EXTENSION}`;
+}
+
+export function parseManifestStartTime(name: string): number | undefined {
+    if (!name.endsWith(MANIFEST_EXTENSION)) return undefined;
+    const parts = name.slice(0, -MANIFEST_EXTENSION.length).split("_");
+    if (parts[0] !== "manifest") return undefined;
+    const startTime = parseInt(parts[1], 10);
+    return Number.isFinite(startTime) ? startTime : undefined;
+}
+
+// Picks the authoritative manifest: newest startTime, ties broken by name for determinism. Returns
+// undefined if there are none (callers then treat every bulk file as valid).
+export function chooseManifest(manifests: { name: string; manifest: Manifest }[]): { name: string; manifest: Manifest } | undefined {
+    let chosen: { name: string; manifest: Manifest } | undefined;
+    for (const entry of manifests) {
+        if (!chosen
+            || entry.manifest.startTime > chosen.manifest.startTime
+            || (entry.manifest.startTime === chosen.manifest.startTime && entry.name > chosen.name)) {
+            chosen = entry;
+        }
+    }
+    return chosen;
+}

package/storage/BulkDatabase2/mergeLock.d.ts

@@ -0,0 +1,2 @@
+export declare function tryAcquireMergeLock(collection: string, holderId: string): boolean;
+export declare function releaseMergeLock(collection: string, holderId: string): void;