sliftutils 1.2.38 → 1.3.0

package/storage/BulkDatabase2/BulkDatabaseFormat.d.ts

@@ -0,0 +1,24 @@
+/// <reference types="node" />
+/// <reference types="node" />
+export declare const KEY_COLUMN = "key";
+export declare const EMPTY_BUFFER: Buffer;
+export declare function buildFileBuffer(rows: Record<string, unknown>[]): Buffer[];
+export type BaseBulkDatabaseReader = {
+    rowCount: number;
+    totalBytes: number;
+    keys: string[];
+    columns: {
+        column: string;
+        byteSize: number;
+    }[];
+    deletedKeys?: Set<string>;
+    getColumn: (column: string) => Promise<{
+        key: string;
+        value: unknown;
+    }[]>;
+    getSingleField: (key: string, column: string) => Promise<unknown | undefined>;
+};
+export declare function loadBulkDatabase(config: {
+    totalBytes: number;
+    getRange: (start: number, end: number) => Promise<Buffer>;
+}): Promise<BaseBulkDatabaseReader>;

package/storage/BulkDatabase2/BulkDatabaseFormat.ts

@@ -0,0 +1,285 @@
+// File format (all integers little-endian):
+//   u32 headerLength, then headerLength bytes of JSON: { rowCount, columns: [{ name, offset, length }] }, where offset is relative to the end of the header.
+//   Each column blob is: u32 offsets[rowCount + 1] (byte offsets into the column's data section), u8 types[rowCount], then the data section (each value's bytes, concatenated).
+// Values are encoded with an explicit type tag per value (see the TYPE_ constants). Reads only fetch the byte ranges they need.
+
+export const KEY_COLUMN = "key";
+
+// Every tag is an explicit constant baked into files on disk. NEVER renumber an existing tag — only add new ones — or every previously written file silently decodes wrong.
+const TYPE_UNDEFINED = 0;
+const TYPE_STRING = 1;
+const TYPE_NUMBER = 2;
+const TYPE_BOOLEAN = 3;
+const TYPE_OBJECT = 4;
+const TYPE_INT8_ARRAY = 5;
+const TYPE_UINT8_ARRAY = 6;
+const TYPE_UINT8_CLAMPED_ARRAY = 7;
+const TYPE_INT16_ARRAY = 8;
+const TYPE_UINT16_ARRAY = 9;
+const TYPE_INT32_ARRAY = 10;
+const TYPE_UINT32_ARRAY = 11;
+const TYPE_FLOAT32_ARRAY = 12;
+const TYPE_FLOAT64_ARRAY = 13;
+
+const TYPED_ARRAY_TYPES: { type: number; ctor: { new(buffer: ArrayBuffer): ArrayBufferView; BYTES_PER_ELEMENT: number; name: string } }[] = [
+    { type: TYPE_INT8_ARRAY, ctor: Int8Array },
+    { type: TYPE_UINT8_ARRAY, ctor: Uint8Array },
+    { type: TYPE_UINT8_CLAMPED_ARRAY, ctor: Uint8ClampedArray },
+    { type: TYPE_INT16_ARRAY, ctor: Int16Array },
+    { type: TYPE_UINT16_ARRAY, ctor: Uint16Array },
+    { type: TYPE_INT32_ARRAY, ctor: Int32Array },
+    { type: TYPE_UINT32_ARRAY, ctor: Uint32Array },
+    { type: TYPE_FLOAT32_ARRAY, ctor: Float32Array },
+    { type: TYPE_FLOAT64_ARRAY, ctor: Float64Array },
+];
+
+export const EMPTY_BUFFER = Buffer.alloc(0) as Buffer;
+
+type FileHeader = {
+    rowCount: number;
+    columns: { name: string; offset: number; length: number }[];
+};
+
+function encodeValue(value: unknown): { type: number; bytes: Buffer } {
+    if (value === undefined || value === null) {
+        return { type: TYPE_UNDEFINED, bytes: EMPTY_BUFFER };
+    }
+    if (typeof value === "string") {
+        return { type: TYPE_STRING, bytes: Buffer.from(value, "utf8") };
+    }
+    if (typeof value === "number") {
+        const bytes = Buffer.alloc(8);
+        bytes.writeDoubleLE(value, 0);
+        return { type: TYPE_NUMBER, bytes };
+    }
+    if (typeof value === "boolean") {
+        return { type: TYPE_BOOLEAN, bytes: Buffer.from([value && 1 || 0]) };
+    }
+    if (ArrayBuffer.isView(value)) {
+        if (value instanceof DataView) {
+            throw new Error(`DataView values are not supported, store a typed array instead`);
+        }
+        const entry = TYPED_ARRAY_TYPES.find(t => value instanceof t.ctor);
+        if (!entry) {
+            throw new Error(`Unsupported typed array type ${value.constructor.name}`);
+        }
+        return {
+            type: entry.type,
+            bytes: Buffer.from(value.buffer, value.byteOffset, value.byteLength),
+        };
+    }
+    if (typeof value !== "object") {
+        throw new Error(`Unsupported value type ${typeof value}`);
+    }
+    return { type: TYPE_OBJECT, bytes: Buffer.from(JSON.stringify(value), "utf8") };
+}
+
+function decodeValue(type: number, bytes: Buffer): unknown {
+    if (type === TYPE_UNDEFINED) return undefined;
+    if (type === TYPE_STRING) return bytes.toString("utf8");
+    if (type === TYPE_NUMBER) {
+        if (bytes.length !== 8) {
+            throw new Error(`Expected 8 bytes for a number, was ${bytes.length}`);
+        }
+        return bytes.readDoubleLE(0);
+    }
+    if (type === TYPE_BOOLEAN) {
+        if (bytes.length !== 1) {
+            throw new Error(`Expected 1 byte for a boolean, was ${bytes.length}`);
+        }
+        return bytes[0] === 1;
+    }
+    if (type === TYPE_OBJECT) return JSON.parse(bytes.toString("utf8"));
+    const entry = TYPED_ARRAY_TYPES.find(t => t.type === type);
+    if (!entry) {
+        throw new Error(`Expected a valid type tag, was ${type}`);
+    }
+    const ctor = entry.ctor;
+    if (type === TYPE_UINT8_ARRAY) return Buffer.from(bytes);
+    if (bytes.length % ctor.BYTES_PER_ELEMENT !== 0) {
+        throw new Error(`Expected byte length divisible by ${ctor.BYTES_PER_ELEMENT} for ${ctor.name}, was ${bytes.length}`);
+    }
+    // Copy to a fresh ArrayBuffer so the typed array view is aligned regardless of where the bytes landed in the source buffer.
+    const aligned = new ArrayBuffer(bytes.length);
+    new Uint8Array(aligned).set(bytes);
+    return new ctor(aligned);
+}
+
+function encodeBulkData(data: unknown[]): Buffer {
+    const n = data.length;
+    const offsets = Buffer.alloc(4 * (n + 1));
+    const types = Buffer.alloc(n);
+    const parts: Buffer[] = [];
+    let pos = 0;
+    for (let i = 0; i < n; i++) {
+        const { type, bytes } = encodeValue(data[i]);
+        offsets.writeUInt32LE(pos, 4 * i);
+        types[i] = type;
+        parts.push(bytes);
+        pos += bytes.length;
+    }
+    offsets.writeUInt32LE(pos, 4 * n);
+    return Buffer.concat([offsets, types, ...parts]);
+}
+
+function decodeBulkData(blob: Buffer, rowCount: number): unknown[] {
+    const indexSize = 4 * (rowCount + 1) + rowCount;
+    const values: unknown[] = [];
+    for (let i = 0; i < rowCount; i++) {
+        const start = blob.readUInt32LE(4 * i);
+        const end = blob.readUInt32LE(4 * (i + 1));
+        const type = blob[4 * (rowCount + 1) + i];
+        values.push(decodeValue(type, blob.subarray(indexSize + start, indexSize + end)));
+    }
+    return values;
+}
+
+// Past this estimated logical size a single rows array is split across multiple files, so no one
+// file (and therefore no single column blob, and no single Buffer.concat) ever approaches the
+// ~2GB Buffer length limit. The cap is deliberately the same as the merge cap: 800MB is the most
+// we'll ever hold for one file. The estimate is rough on purpose — it only needs to keep each
+// chunk comfortably under the limit, not be exact.
+const FILE_SPLIT_BYTES = 800 * 1024 * 1024;
+
+// Per-value on-disk overhead: a 4-byte offset entry plus a 1-byte type tag.
+const PER_VALUE_OVERHEAD = 5;
+
+function estimateValueBytes(value: unknown): number {
+    if (value === undefined || value === null) return 0;
+    if (typeof value === "string") return value.length * 2;
+    if (typeof value === "number") return 8;
+    if (typeof value === "boolean") return 1;
+    if (ArrayBuffer.isView(value)) return value.byteLength;
+    if (typeof value === "object") return JSON.stringify(value).length;
+    return 0;
+}
+
+function estimateRowBytes(row: Record<string, unknown>): number {
+    let total = 0;
+    for (const value of Object.values(row)) {
+        total += estimateValueBytes(value) + PER_VALUE_OVERHEAD;
+    }
+    return total;
+}
+
+function buildOneFile(rows: Record<string, unknown>[]): Buffer {
+    const columnNames: string[] = [];
+    const columnSet = new Set<string>();
+    for (const row of rows) {
+        for (const field of Object.keys(row)) {
+            if (columnSet.has(field)) continue;
+            columnSet.add(field);
+            columnNames.push(field);
+        }
+    }
+    const blobs = columnNames.map(col => encodeBulkData(rows.map(row => row[col])));
+    let offset = 0;
+    const columns = columnNames.map((name, i) => {
+        const entry = { name, offset, length: blobs[i].length };
+        offset += blobs[i].length;
+        return entry;
+    });
+    const header: FileHeader = { rowCount: rows.length, columns };
+    const headerBuf = Buffer.from(JSON.stringify(header), "utf8");
+    const lengthPrefix = Buffer.alloc(4);
+    lengthPrefix.writeUInt32LE(headerBuf.length, 0);
+    return Buffer.concat([lengthPrefix, headerBuf, ...blobs]);
+}
+
+// Returns one complete, independent file buffer per chunk of rows. When the caller hands us more
+// rows than fit comfortably in one file we partition by row range — each returned buffer is exactly
+// what buildOneFile would produce if called with that subset, so the chunks have disjoint keys and
+// the caller just writes each as its own file. A normal-sized write returns a single buffer.
+export function buildFileBuffer(rows: Record<string, unknown>[]): Buffer[] {
+    if (rows.length === 0) return [buildOneFile([])];
+    const result: Buffer[] = [];
+    let chunkStart = 0;
+    let chunkBytes = 0;
+    for (let i = 0; i < rows.length; i++) {
+        const rowBytes = estimateRowBytes(rows[i]);
+        if (i > chunkStart && chunkBytes + rowBytes > FILE_SPLIT_BYTES) {
+            result.push(buildOneFile(rows.slice(chunkStart, i)));
+            chunkStart = i;
+            chunkBytes = 0;
+        }
+        chunkBytes += rowBytes;
+    }
+    result.push(buildOneFile(rows.slice(chunkStart)));
+    return result;
+}
+
+export type BaseBulkDatabaseReader = {
+    rowCount: number;
+    totalBytes: number;
+    // Keys is special, it's always automatically decoded, even though it is stored as a normal column
+    keys: string[];
+    columns: { column: string; byteSize: number }[];
+    // Keys this reader tombstones (deleted). A newer reader's deletion suppresses the key in all
+    // older readers. Bulk readers never set this; the tier-0 stream reader does.
+    deletedKeys?: Set<string>;
+    getColumn: (column: string) => Promise<{
+        key: string;
+        value: unknown;
+    }[]>;
+    getSingleField: (key: string, column: string) => Promise<unknown | undefined>;
+};
+
+export async function loadBulkDatabase(config: {
+    totalBytes: number;
+    getRange: (start: number, end: number) => Promise<Buffer>;
+}): Promise<BaseBulkDatabaseReader> {
+    const headerLength = (await config.getRange(0, 4)).readUInt32LE(0);
+    if (headerLength <= 0 || headerLength > config.totalBytes) {
+        throw new Error(`Expected header length in (0, ${config.totalBytes}], was ${headerLength}`);
+    }
+    const header = JSON.parse((await config.getRange(4, 4 + headerLength)).toString("utf8")) as FileHeader;
+    const dataBase = 4 + headerLength;
+    const rowCount = header.rowCount;
+    const colByName = new Map(header.columns.map(c => [c.name, c]));
+
+    async function readWholeColumn(column: string): Promise<unknown[]> {
+        const col = colByName.get(column);
+        if (!col) {
+            throw new Error(`Expected column ${column}, file only has: ${header.columns.map(c => c.name).join(", ")}`);
+        }
+        const blob = await config.getRange(dataBase + col.offset, dataBase + col.offset + col.length);
+        return decodeBulkData(blob, rowCount);
+    }
+
+    const keys = (await readWholeColumn(KEY_COLUMN)).map(v => {
+        if (typeof v !== "string") {
+            throw new Error(`Expected string key, was ${typeof v}: ${JSON.stringify(v)?.slice(0, 500)}`);
+        }
+        return v;
+    });
+    const keyIndex = new Map(keys.map((key, i) => [key, i]));
+
+    return {
+        rowCount,
+        totalBytes: config.totalBytes,
+        keys,
+        columns: header.columns.map(c => ({ column: c.name, byteSize: c.length })),
+        async getColumn(column) {
+            const values = await readWholeColumn(column);
+            return keys.map((key, i) => ({ key, value: values[i] }));
+        },
+        async getSingleField(key, column) {
+            const row = keyIndex.get(key);
+            if (row === undefined) return undefined;
+            const col = colByName.get(column);
+            if (!col) return undefined;
+            const colBase = dataBase + col.offset;
+            const offsetsBuf = await config.getRange(colBase + 4 * row, colBase + 4 * row + 8);
+            const start = offsetsBuf.readUInt32LE(0);
+            const end = offsetsBuf.readUInt32LE(4);
+            const typePos = colBase + 4 * (rowCount + 1) + row;
+            const typeBuf = await config.getRange(typePos, typePos + 1);
+            const dataStart = colBase + 4 * (rowCount + 1) + rowCount;
+            let bytes = EMPTY_BUFFER;
+            if (end > start) {
+                bytes = await config.getRange(dataStart + start, dataStart + end);
+            }
+            return decodeValue(typeBuf[0], bytes);
+        },
+    };
+}

package/storage/BulkDatabase2/blockCache.d.ts

@@ -0,0 +1,17 @@
+/// <reference types="node" />
+/// <reference types="node" />
+export type GetRange = (start: number, end: number) => Promise<Buffer>;
+export declare function encodeCompressedBlocks(data: Buffer): Buffer;
+export declare class BlockCache {
+    private blocks;
+    private indexes;
+    clear(): void;
+    private touch;
+    private readIndex;
+    open(fileId: string, fileSize: number, rawGetRange: GetRange): Promise<{
+        uncompressedSize: number;
+        getRange: GetRange;
+    }>;
+    private makeGetRange;
+}
+export declare const blockCache: BlockCache;

package/storage/BulkDatabase2/blockCache.ts

@@ -0,0 +1,173 @@
+import { LZ4 } from "socket-function/src/lz4/LZ4";
+
+// Block-aligned, promise-deduped, decompressing range cache for BulkDatabase2's on-disk files.
+//
+// On-disk format (produced by encodeCompressedBlocks): the logical (uncompressed) bulk buffer is
+// split into fixed BLOCK_SIZE blocks; each block is LZ4-compressed unless that wouldn't shrink it by
+// at least 2x, in which case it's stored raw. The file starts with a JSON index mapping each block to
+// its stored length + a compressed flag, so a reader can seek straight to any block:
+//
+//   [u32 indexLength][index JSON][block 0 stored bytes][block 1 stored bytes]...
+//
+// The cache presents a *logical* getRange (over uncompressed bytes) so BulkDatabase2's reader is
+// oblivious to compression. It reads compressed bytes via the underlying getRange, decompresses, and
+// caches the uncompressed blocks. Reads are promise-deduped and contiguous missing blocks are
+// coalesced into one underlying read. Files are immutable, so cached blocks are valid forever.
+//
+// Per-block compression mainly helps slow storage (HDD): fewer bytes off disk per block, at the cost
+// of a fast in-memory LZ4 decompress.
+
+const BLOCK_SIZE = 256 * 1024;
+const MAX_BLOCKS = Math.floor((512 * 1024 * 1024) / BLOCK_SIZE);
+const MIN_COMPRESSION_RATIO = 2;
+
+const EMPTY = Buffer.alloc(0) as Buffer;
+
+export type GetRange = (start: number, end: number) => Promise<Buffer>;
+
+type BlockMeta = { len: number; c: 0 | 1 };
+type FileIndex = {
+    uncompressedSize: number;
+    blockSize: number;
+    blocks: BlockMeta[];
+    // Stored byte offset of each block (offsets[i]..offsets[i]+blocks[i].len), filled in on parse.
+    offsets: number[];
+};
+
+// Writes a logical bulk buffer in the compressed-block format described above.
+export function encodeCompressedBlocks(data: Buffer): Buffer {
+    let blocks: BlockMeta[] = [];
+    let storedParts: Buffer[] = [];
+    let blockCount = Math.ceil(data.length / BLOCK_SIZE);
+    for (let i = 0; i < blockCount; i++) {
+        let raw = data.subarray(i * BLOCK_SIZE, Math.min((i + 1) * BLOCK_SIZE, data.length));
+        let compressed = LZ4.compress(raw);
+        // Only keep the compressed form if it shrinks the block by at least MIN_COMPRESSION_RATIO.
+        if (compressed.length * MIN_COMPRESSION_RATIO <= raw.length) {
+            storedParts.push(compressed);
+            blocks.push({ len: compressed.length, c: 1 });
+        } else {
+            storedParts.push(raw);
+            blocks.push({ len: raw.length, c: 0 });
+        }
+    }
+    let indexBuf = Buffer.from(JSON.stringify({ uncompressedSize: data.length, blockSize: BLOCK_SIZE, blocks }), "utf8");
+    let prefix = Buffer.alloc(4);
+    prefix.writeUInt32LE(indexBuf.length, 0);
+    return Buffer.concat([prefix, indexBuf, ...storedParts]);
+}
+
+export class BlockCache {
+    // Uncompressed blocks; insertion order is LRU order (re-inserted on access).
+    private blocks = new Map<string, Promise<Buffer>>();
+    // Parsed file index per fileId (small; not subject to the block LRU).
+    private indexes = new Map<string, Promise<FileIndex>>();
+
+    public clear() {
+        this.blocks.clear();
+        this.indexes.clear();
+    }
+
+    private touch(key: string, value: Promise<Buffer>) {
+        this.blocks.delete(key);
+        this.blocks.set(key, value);
+        while (this.blocks.size > MAX_BLOCKS) {
+            let oldest = this.blocks.keys().next().value;
+            if (oldest === undefined) break;
+            this.blocks.delete(oldest);
+        }
+    }
+
+    // Reads + validates the file index. Rejects the WHOLE file if it isn't exactly the size its index
+    // implies — a truncated/partial write (e.g. a crash mid-write) is detected here rather than
+    // silently returning corrupted values for the rows near the end.
+    private async readIndex(rawGetRange: GetRange, fileSize: number): Promise<FileIndex> {
+        let head = await rawGetRange(0, 4);
+        if (head.length < 4) throw new Error(`bulk file too short for an index header (${fileSize} bytes)`);
+        let indexLength = head.readUInt32LE(0);
+        if (indexLength <= 0 || 4 + indexLength > fileSize) {
+            throw new Error(`bulk file index length ${indexLength} is invalid for a ${fileSize}-byte file`);
+        }
+        let indexBuf = await rawGetRange(4, 4 + indexLength);
+        let parsed = JSON.parse(indexBuf.toString("utf8")) as { uncompressedSize: number; blockSize: number; blocks: BlockMeta[] };
+        let dataBase = 4 + indexLength;
+        let offsets: number[] = [];
+        let offset = dataBase;
+        for (let block of parsed.blocks) {
+            offsets.push(offset);
+            offset += block.len;
+        }
+        // The blocks must account for exactly the rest of the file. Any mismatch means the file is
+        // truncated or otherwise corrupt — reject it entirely.
+        if (offset !== fileSize) {
+            throw new Error(`bulk file is ${fileSize} bytes but its index implies ${offset} (truncated/corrupt)`);
+        }
+        return { ...parsed, offsets };
+    }
+
+    // Opens an immutable file: reads + validates its index (cached) and returns the logical
+    // (uncompressed) size plus a logical getRange that the caller can use exactly like an
+    // uncompressed file. `fileSize` is the actual on-disk byte length, used to detect truncation.
+    public async open(fileId: string, fileSize: number, rawGetRange: GetRange): Promise<{ uncompressedSize: number; getRange: GetRange }> {
+        let indexPromise = this.indexes.get(fileId);
+        if (!indexPromise) {
+            indexPromise = this.readIndex(rawGetRange, fileSize);
+            this.indexes.set(fileId, indexPromise);
+        }
+        let index = await indexPromise;
+        return { uncompressedSize: index.uncompressedSize, getRange: this.makeGetRange(fileId, index, rawGetRange) };
+    }
+
+    private makeGetRange(fileId: string, index: FileIndex, rawGetRange: GetRange): GetRange {
+        let blockSize = index.blockSize;
+        return async (start, end) => {
+            if (end <= start) return EMPTY;
+            let firstBlock = Math.floor(start / blockSize);
+            let lastBlock = Math.floor((end - 1) / blockSize);
+
+            let parts: Buffer[] = [];
+            let block = firstBlock;
+            while (block <= lastBlock) {
+                let key = `${fileId}:${block}`;
+                let cached = this.blocks.get(key);
+                if (cached) {
+                    this.touch(key, cached);
+                    parts.push(await cached);
+                    block++;
+                    continue;
+                }
+                // Coalesce a run of consecutive missing blocks into one underlying (compressed) read.
+                let runStart = block;
+                let runEnd = block + 1;
+                while (runEnd <= lastBlock && !this.blocks.has(`${fileId}:${runEnd}`)) runEnd++;
+                let compStart = index.offsets[runStart];
+                let lastMeta = index.blocks[runEnd - 1];
+                let compEnd = index.offsets[runEnd - 1] + lastMeta.len;
+                let runCompressed = rawGetRange(compStart, compEnd);
+                // Register a per-block promise BEFORE awaiting so concurrent reads dedupe onto it.
+                for (let i = runStart; i < runEnd; i++) {
+                    let meta = index.blocks[i];
+                    let relOffset = index.offsets[i] - compStart;
+                    let blockPromise = runCompressed.then(buf => {
+                        let stored = buf.subarray(relOffset, relOffset + meta.len);
+                        if (meta.c) return LZ4.decompress(stored);
+                        return stored;
+                    });
+                    this.touch(`${fileId}:${i}`, blockPromise);
+                }
+                for (let i = runStart; i < runEnd; i++) {
+                    let promise = this.blocks.get(`${fileId}:${i}`);
+                    if (promise) parts.push(await promise);
+                }
+                block = runEnd;
+            }
+
+            let combined = parts.length === 1 && parts[0] || Buffer.concat(parts);
+            let sliceStart = start - firstBlock * blockSize;
+            return combined.subarray(sliceStart, sliceStart + (end - start));
+        };
+    }
+}
+
+// Shared across every BulkDatabase2 collection/file (the 512MB budget is global).
+export const blockCache = new BlockCache();

package/storage/BulkDatabase2/streamLog.d.ts

@@ -0,0 +1,25 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { BaseBulkDatabaseReader } from "./BulkDatabaseFormat";
+export declare const STREAM_EXTENSION = ".stream";
+export type StreamEntry = {
+    time: number;
+    row?: Record<string, unknown>;
+    deletedKey?: string;
+};
+export declare function frameRows(entries: {
+    time: number;
+    row: Record<string, unknown>;
+}[]): Buffer;
+export declare function frameDeletes(entries: {
+    time: number;
+    key: string;
+}[]): Buffer;
+export declare function parseStream(buffer: Buffer): {
+    entries: StreamEntry[];
+    badBytes: number;
+};
+export declare function streamReaderFromEntries(entries: StreamEntry[], totalBytes: number): {
+    reader: BaseBulkDatabaseReader;
+    times: Map<string, number>;
+};

package/storage/BulkDatabase2/streamLog.ts

@@ -0,0 +1,116 @@
+import cborx from "cbor-x";
+import { BaseBulkDatabaseReader } from "./BulkDatabaseFormat";
+
+// Tier-0 streaming format: an append log of whole-row writes and deletes (row-format, not columnar),
+// so small mutations are a single cheap append instead of rewriting a columnar file. Each block is:
+//
+//   [u32 len][CBOR({ t, v })  or  CBOR({ t, d })][u32 len]
+//
+// `t` is a per-write unique timestamp (getTimeUnique); `v` is a set row; `d` is a deleted key. Because
+// every thread streams to its own file, the only way to recover the global mutation order — needed
+// for newest-wins — is a per-write timestamp, with ties broken by file name. The trailing length must
+// match the leading one; a torn/incomplete append leaves a mismatched or missing suffix, so we stop
+// there and report trailing bad bytes instead of throwing. structuredClone preserves typed arrays.
+
+export const STREAM_EXTENSION = ".stream";
+
+const cborEncoder = new cborx.Encoder({ structuredClone: true });
+
+export type StreamEntry = { time: number; row?: Record<string, unknown>; deletedKey?: string };
+
+function frame(payload: Buffer): Buffer {
+    let len = Buffer.alloc(4);
+    len.writeUInt32LE(payload.length, 0);
+    return Buffer.concat([len, payload, len]);
+}
+
+// We deliberately do NOT batch/debounce writes here — each write is framed and appended (and flushed)
+// immediately. If a caller makes many individual writes and that causes lag, the fix is for them to
+// call writeBatch/deleteBatch with the whole set, not for us to silently coalesce. Caller-side
+// batching is strictly faster than anything we could do (it knows the full set up front), and not
+// batching gives the lowest possible latency for callers who genuinely want single writes — which
+// matters because the tab can be closed at any moment, so we want each write on disk as soon as
+// possible rather than sitting in a pending buffer that a close would lose.
+
+// Times are assigned by the caller (BulkDatabase2) so the exact same timestamp lands on disk, in the
+// in-memory overlay, and in the cross-tab broadcast — keeping the global write order consistent.
+export function frameRows(entries: { time: number; row: Record<string, unknown> }[]): Buffer {
+    return Buffer.concat(entries.map(e => frame(Buffer.from(cborEncoder.encode({ t: e.time, v: e.row })))));
+}
+
+export function frameDeletes(entries: { time: number; key: string }[]): Buffer {
+    return Buffer.concat(entries.map(e => frame(Buffer.from(cborEncoder.encode({ t: e.time, d: e.key })))));
+}
+
+export function parseStream(buffer: Buffer): { entries: StreamEntry[]; badBytes: number } {
+    let entries: StreamEntry[] = [];
+    let pos = 0;
+    while (pos + 4 <= buffer.length) {
+        let len = buffer.readUInt32LE(pos);
+        let payloadStart = pos + 4;
+        let suffixStart = payloadStart + len;
+        if (suffixStart + 4 > buffer.length) break;
+        if (buffer.readUInt32LE(suffixStart) !== len) break;
+        let decoded: { t: number; v?: Record<string, unknown>; d?: string } | undefined;
+        try {
+            decoded = cborEncoder.decode(buffer.subarray(payloadStart, suffixStart));
+        } catch {
+            break;
+        }
+        if (decoded && decoded.d !== undefined) entries.push({ time: decoded.t, deletedKey: decoded.d });
+        else if (decoded && decoded.v) entries.push({ time: decoded.t, row: decoded.v });
+        pos = suffixStart + 4;
+    }
+    return { entries, badBytes: buffer.length - pos };
+}
+
+// Wraps streamed entries (already ordered oldest-first) as a BaseBulkDatabaseReader. Applies
+// set/delete newest-wins: later entries overwrite earlier ones, and a delete tombstones the key
+// (exposed via deletedKeys so the join suppresses it in older bulk readers). Also returns the latest
+// timestamp seen per key (live or deleted), used for cross-tab conflict resolution.
+export function streamReaderFromEntries(entries: StreamEntry[], totalBytes: number): { reader: BaseBulkDatabaseReader; times: Map<string, number> } {
+    let byKey = new Map<string, Record<string, unknown>>();
+    let deletedKeys = new Set<string>();
+    let times = new Map<string, number>();
+    for (let entry of entries) {
+        if (entry.deletedKey !== undefined) {
+            byKey.delete(entry.deletedKey);
+            deletedKeys.add(entry.deletedKey);
+            times.set(entry.deletedKey, entry.time);
+        } else if (entry.row) {
+            let key = entry.row.key as string;
+            byKey.set(key, entry.row);
+            deletedKeys.delete(key);
+            times.set(key, entry.time);
+        }
+    }
+    let keys = [...byKey.keys()];
+    let columnNames: string[] = [];
+    let seen = new Set<string>();
+    for (let row of byKey.values()) {
+        for (let field of Object.keys(row)) {
+            if (seen.has(field)) continue;
+            seen.add(field);
+            columnNames.push(field);
+        }
+    }
+    let columns = columnNames.map(column => ({ column, byteSize: 0 }));
+    let reader: BaseBulkDatabaseReader = {
+        totalBytes,
+        rowCount: keys.length,
+        keys,
+        columns,
+        deletedKeys,
+        async getColumn(column) {
+            return keys.map(key => {
+                let row = byKey.get(key);
+                return { key, value: row && row[column] };
+            });
+        },
+        async getSingleField(key, column) {
+            let row = byKey.get(key);
+            return row && row[column];
+        },
+    };
+    return { reader, times };
+}

package/storage/BulkDatabase2/syncClient.d.ts

@@ -0,0 +1,9 @@
+export type RemoteWrite = {
+    key: string;
+    time: number;
+    deleted?: boolean;
+    value?: unknown;
+};
+export declare function isSyncSupported(): boolean;
+export declare function connect(collection: string, onWrite: (write: RemoteWrite) => void): Promise<RemoteWrite[]>;
+export declare function broadcast(collection: string, write: RemoteWrite): void;