sliftutils 1.4.47 → 1.4.49

package/storage/BulkDatabase2/BulkDatabaseFormat.d.ts

@@ -3,8 +3,43 @@
 export declare const KEY_COLUMN = "key";
 export declare const EMPTY_BUFFER: Buffer;
 export declare const ABSENT: unique symbol;
+export type RawCell = {
+    type: number;
+    bytes: Buffer;
+};
+export declare const TYPE_ABSENT_TAG = 14;
+export type ColumnIndex = {
+    offsets: Uint32Array;
+    types: Uint8Array;
+    readValueBytes: (startRow: number, endRow: number) => Promise<Buffer>;
+};
+export declare function encodeValue(value: unknown): {
+    type: number;
+    bytes: Buffer;
+};
 export declare const TARGET_FILE_BYTES: number;
-export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes?: number): Buffer[];
+export type RawRow = {
+    key: string;
+    time: number;
+    cells: Map<string, RawCell>;
+};
+export declare function columnIndexByteLength(rowCount: number): number;
+export declare function assemblePlannedFile(config: {
+    valueColumns: {
+        name: string;
+        blob: Buffer;
+    }[];
+    keys: string[];
+    times: number[];
+}): Buffer;
+export interface BuiltFile {
+    buffer: Buffer;
+    minKey: string;
+    maxKey: string;
+    rowCount: number;
+}
+export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes?: number): BuiltFile[];
+export declare function buildFileBufferRaw(rows: RawRow[], targetBytes?: number): BuiltFile[];
 export type BaseBulkDatabaseReader = {
     rowCount: number;
     totalBytes: number;
@@ -24,6 +59,9 @@ export type BaseBulkDatabaseReader = {
         value: unknown;
         time: number;
     }[]>;
+    getRawColumn: (column: string) => Promise<Map<string, RawCell>>;
+    getColumnIndex: (column: string) => Promise<ColumnIndex>;
+    rowOfKey: (key: string) => number | undefined;
     getSingleField: (key: string, column: string) => Promise<{
         value: unknown;
         time: number;

package/storage/BulkDatabase2/BulkDatabaseFormat.ts

@@ -43,6 +43,35 @@ export const EMPTY_BUFFER = Buffer.alloc(0) as Buffer;
 // to an older reader for that column. Distinct from a stored undefined, which is a real clearing value.
 export const ABSENT = Symbol("absent");
 
+// One value exactly as it sits on disk: its type tag + raw bytes. A cell's encoding is position-
+// independent (nothing about it depends on which row/file it lives in), so a merge copies the winning
+// RawCell straight from an input column into the output — never decoding it to a JS value and
+// re-encoding. That's far less memory (no object materialization) and much faster (a byte copy, not a
+// JPEG/typed-array decode + re-encode).
+export type RawCell = { type: number; bytes: Buffer };
+
+// The type tag a cell carries when its row never set this column — read it from the column's `types`
+// array to detect ABSENT without decoding. Re-exported so the merge planner (which works at the
+// column-index level, no value materialization) can check it.
+export const TYPE_ABSENT_TAG = TYPE_ABSENT;
+
+// A column's index alone (offsets + types — small, ~5 bytes/row) plus a primitive to read a CONTIGUOUS
+// row range's raw value bytes. Used by the planned merge: it loads the index across every (source,
+// column) once (cheap), uses types to detect ABSENT and offsets to size each cell, plans the byte
+// layout of every output file from those — then in the execute phase reads only the runs of bytes it
+// actually needs, copying them straight into pre-laid-out output buffers. No cell value is ever
+// materialized as a JS object.
+export type ColumnIndex = {
+    // offsets[i]..offsets[i+1] is row i's value byte range within the column's data section.
+    offsets: Uint32Array;
+    // Per-row type tag; TYPE_ABSENT_TAG marks "this row never set this column" (fall-through).
+    types: Uint8Array;
+    // Read the contiguous bytes of value(s) for rows [startRow, endRow). Length is exactly
+    // offsets[endRow] - offsets[startRow] — the cells in that range concatenated. The caller composes
+    // larger reads from consecutive rows (one source-side getRange per run).
+    readValueBytes: (startRow: number, endRow: number) => Promise<Buffer>;
+};
+
 // Hidden per-row column holding each row's write-time (so reads can resolve a key to its latest value
 // by actual time). NUL-prefixed so it can't collide with a user column; excluded from `columns`.
 const TIME_COLUMN = String.fromCharCode(0) + "t";
@@ -62,7 +91,7 @@ type FileHeader = {
     maxKey?: string;
 };
 
-function encodeValue(value: unknown): { type: number; bytes: Buffer } {
+export function encodeValue(value: unknown): { type: number; bytes: Buffer } {
     if (value === ABSENT) {
         return { type: TYPE_ABSENT, bytes: EMPTY_BUFFER };
     }
@@ -148,6 +177,26 @@ function encodeBulkData(data: unknown[]): Buffer {
     return Buffer.concat([offsets, types, ...parts]);
 }
 
+// Like encodeBulkData but the values are already encoded (a merge supplies the winning cells' raw bytes
+// straight from the input files). Lays out the same offsets / types / data section without touching the
+// values themselves.
+function encodeBulkDataRaw(cells: RawCell[]): Buffer {
+    const n = cells.length;
+    const offsets = Buffer.alloc(4 * (n + 1));
+    const types = Buffer.alloc(n);
+    const parts: Buffer[] = [];
+    let pos = 0;
+    for (let i = 0; i < n; i++) {
+        const { type, bytes } = cells[i];
+        offsets.writeUInt32LE(pos, 4 * i);
+        types[i] = type;
+        parts.push(bytes);
+        pos += bytes.length;
+    }
+    offsets.writeUInt32LE(pos, 4 * n);
+    return Buffer.concat([offsets, types, ...parts]);
+}
+
 function decodeBulkData(blob: Buffer, rowCount: number): unknown[] {
     const indexSize = 4 * (rowCount + 1) + rowCount;
     const values: unknown[] = [];
@@ -188,21 +237,10 @@ function estimateRowBytes(row: Record<string, unknown>): number {
     return total;
 }
 
-function buildOneFile(rows: Record<string, unknown>[], times: number[]): Buffer {
-    const columnNames: string[] = [];
-    const columnSet = new Set<string>();
-    for (const row of rows) {
-        for (const field of Object.keys(row)) {
-            if (columnSet.has(field)) continue;
-            columnSet.add(field);
-            columnNames.push(field);
-        }
-    }
-    // A row that doesn't include a column stores ABSENT (fall-through), not undefined (a real value).
-    const blobs = columnNames.map(col => encodeBulkData(rows.map(row => col in row ? row[col] : ABSENT)));
-    // The hidden per-row time column.
-    columnNames.push(TIME_COLUMN);
-    blobs.push(encodeBulkData(times));
+// Concatenates already-encoded column blobs into a complete file (4-byte header length + header JSON +
+// blobs), computing the header's time + key bounds from the per-row times and keys. Shared by the
+// object-based builder (buildOneFile) and the raw-splice builder (buildOneFileRaw).
+function assembleFile(columnNames: string[], blobs: Buffer[], rowCount: number, times: number[], keys: string[]): Buffer {
     let offset = 0;
     const columns = columnNames.map((name, i) => {
         const entry = { name, offset, length: blobs[i].length };
@@ -214,26 +252,111 @@ function buildOneFile(rows: Record<string, unknown>[], times: number[]): Buffer
     for (const t of times) { if (t < minTime) minTime = t; if (t > maxTime) maxTime = t; }
     let minKey: string | undefined;
     let maxKey: string | undefined;
-    for (const row of rows) {
-        const key = row[KEY_COLUMN] as string;
+    for (const key of keys) {
         if (minKey === undefined || key < minKey) minKey = key;
         if (maxKey === undefined || key > maxKey) maxKey = key;
     }
-    const header: FileHeader = { rowCount: rows.length, columns, minTime, maxTime, minKey, maxKey };
+    const header: FileHeader = { rowCount, columns, minTime, maxTime, minKey, maxKey };
     const headerBuf = Buffer.from(JSON.stringify(header), "utf8");
     const lengthPrefix = Buffer.alloc(4);
     lengthPrefix.writeUInt32LE(headerBuf.length, 0);
     return Buffer.concat([lengthPrefix, headerBuf, ...blobs]);
 }
 
-// Returns one complete, independent file buffer per chunk of rows. Rows are first sorted by key, then
+function buildOneFile(rows: Record<string, unknown>[], times: number[]): Buffer {
+    const columnNames: string[] = [];
+    const columnSet = new Set<string>();
+    for (const row of rows) {
+        for (const field of Object.keys(row)) {
+            if (columnSet.has(field)) continue;
+            columnSet.add(field);
+            columnNames.push(field);
+        }
+    }
+    // A row that doesn't include a column stores ABSENT (fall-through), not undefined (a real value).
+    const blobs = columnNames.map(col => encodeBulkData(rows.map(row => col in row ? row[col] : ABSENT)));
+    // The hidden per-row time column.
+    columnNames.push(TIME_COLUMN);
+    blobs.push(encodeBulkData(times));
+    return assembleFile(columnNames, blobs, rows.length, times, rows.map(r => r[KEY_COLUMN] as string));
+}
+
+// A resolved output row for the raw-splice merge: its key, write-time, and the winning raw cell for each
+// column it has (a column it lacks is written ABSENT — fall-through). The cell bytes are copied straight
+// from the input files; no value is ever decoded.
+export type RawRow = { key: string; time: number; cells: Map<string, RawCell> };
+
+// Size of a column blob's INDEX section (offsets array + types array) for a column of N rows. The data
+// section follows immediately after. The planned merge uses this to size output buffers exactly.
+export function columnIndexByteLength(rowCount: number): number {
+    return 4 * (rowCount + 1) + rowCount;
+}
+
+// Assembles a complete bulk file from a set of pre-built value column blobs plus the keys + times. Auto-
+// adds the KEY_COLUMN (keys) and hidden TIME_COLUMN (times) — they're small enough to encode in one shot
+// here. Used by the planned merge: it builds each value column's blob by-hand (offsets/types + raw bytes
+// copied directly from inputs, no value materialization) and hands the result here for header + bounds.
+export function assemblePlannedFile(config: {
+    valueColumns: { name: string; blob: Buffer }[];
+    keys: string[];
+    times: number[];
+}): Buffer {
+    const columnNames = [KEY_COLUMN, ...config.valueColumns.map(c => c.name), TIME_COLUMN];
+    const blobs = [
+        encodeBulkData(config.keys),
+        ...config.valueColumns.map(c => c.blob),
+        encodeBulkData(config.times),
+    ];
+    return assembleFile(columnNames, blobs, config.keys.length, config.times, config.keys);
+}
+
+const ABSENT_CELL: RawCell = { type: TYPE_ABSENT, bytes: EMPTY_BUFFER };
+
+function buildOneFileRaw(rows: RawRow[]): Buffer {
+    // Columns present in this chunk, in first-seen order (matches buildOneFile, which only emits columns
+    // some row actually has).
+    const valueColumns: string[] = [];
+    const seen = new Set<string>();
+    for (const row of rows) for (const col of row.cells.keys()) {
+        if (seen.has(col)) continue;
+        seen.add(col);
+        valueColumns.push(col);
+    }
+    const columnNames = [KEY_COLUMN, ...valueColumns, TIME_COLUMN];
+    const times = rows.map(r => r.time);
+    const blobs = [
+        encodeBulkData(rows.map(r => r.key)),
+        ...valueColumns.map(col => encodeBulkDataRaw(rows.map(r => r.cells.get(col) ?? ABSENT_CELL))),
+        encodeBulkData(times),
+    ];
+    return assembleFile(columnNames, blobs, rows.length, times, rows.map(r => r.key));
+}
+
+function estimateRawRowBytes(row: RawRow): number {
+    let total = row.key.length * 2 + PER_VALUE_OVERHEAD;
+    for (const cell of row.cells.values()) total += cell.bytes.length + PER_VALUE_OVERHEAD;
+    return total;
+}
+
+// One complete, independent file: the encoded buffer plus its key range + row count (the caller logs the
+// range when a merge splits across several files).
+export interface BuiltFile { buffer: Buffer; minKey: string; maxKey: string; rowCount: number; }
+
+// Returns one complete, independent file per chunk of rows. Rows are first sorted by key, then
 // partitioned into key-contiguous chunks of ~targetBytes each — so every returned file is key-sorted
 // (tight minKey/maxKey for the read-skip + merge planner) and stays near the target size, and no
 // single column blob / Buffer.concat approaches the ~2GB limit. The chunks have disjoint key ranges,
-// so the caller just writes each as its own file. A normal-sized write returns a single buffer.
+// so the caller just writes each as its own file. A normal-sized write returns a single file.
 // `times[i]` is row i's write-time, stored per row so reads resolve a key to its latest value by time.
-export function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes = TARGET_FILE_BYTES): Buffer[] {
-    if (rows.length === 0) return [buildOneFile([], [])];
+export function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes = TARGET_FILE_BYTES): BuiltFile[] {
+    // A chunk is already key-sorted, so its first/last row are its min/max key.
+    const make = (rs: Record<string, unknown>[], ts: number[]): BuiltFile => ({
+        buffer: buildOneFile(rs, ts),
+        minKey: rs.length ? (rs[0][KEY_COLUMN] as string) : "",
+        maxKey: rs.length ? (rs[rs.length - 1][KEY_COLUMN] as string) : "",
+        rowCount: rs.length,
+    });
+    if (rows.length === 0) return [make([], [])];
     // Sort rows + their times together by key so each output file is key-contiguous.
     const order = rows.map((_, i) => i).sort((a, b) => {
         const ka = rows[a][KEY_COLUMN] as string;
@@ -242,19 +365,47 @@ export function buildFileBuffer(rows: Record<string, unknown>[], times: number[]
     });
     const sortedRows = order.map(i => rows[i]);
     const sortedTimes = order.map(i => times[i]);
-    const result: Buffer[] = [];
+    const result: BuiltFile[] = [];
     let chunkStart = 0;
     let chunkBytes = 0;
     for (let i = 0; i < sortedRows.length; i++) {
         const rowBytes = estimateRowBytes(sortedRows[i]);
         if (i > chunkStart && chunkBytes + rowBytes > targetBytes) {
-            result.push(buildOneFile(sortedRows.slice(chunkStart, i), sortedTimes.slice(chunkStart, i)));
+            result.push(make(sortedRows.slice(chunkStart, i), sortedTimes.slice(chunkStart, i)));
             chunkStart = i;
             chunkBytes = 0;
         }
         chunkBytes += rowBytes;
     }
-    result.push(buildOneFile(sortedRows.slice(chunkStart), sortedTimes.slice(chunkStart)));
+    result.push(make(sortedRows.slice(chunkStart), sortedTimes.slice(chunkStart)));
+    return result;
+}
+
+// The raw-splice counterpart of buildFileBuffer, used by merges: the rows already carry their winning
+// cells as raw on-disk bytes (no JS values), so this just sorts by key, chunks to ~targetBytes, and
+// concatenates the bytes. Same output guarantees: key-contiguous, disjoint, ascending files.
+export function buildFileBufferRaw(rows: RawRow[], targetBytes = TARGET_FILE_BYTES): BuiltFile[] {
+    const make = (rs: RawRow[]): BuiltFile => ({
+        buffer: buildOneFileRaw(rs),
+        minKey: rs.length ? rs[0].key : "",
+        maxKey: rs.length ? rs[rs.length - 1].key : "",
+        rowCount: rs.length,
+    });
+    if (rows.length === 0) return [make([])];
+    const sorted = rows.slice().sort((a, b) => a.key < b.key ? -1 : a.key > b.key ? 1 : 0);
+    const result: BuiltFile[] = [];
+    let chunkStart = 0;
+    let chunkBytes = 0;
+    for (let i = 0; i < sorted.length; i++) {
+        const rowBytes = estimateRawRowBytes(sorted[i]);
+        if (i > chunkStart && chunkBytes + rowBytes > targetBytes) {
+            result.push(make(sorted.slice(chunkStart, i)));
+            chunkStart = i;
+            chunkBytes = 0;
+        }
+        chunkBytes += rowBytes;
+    }
+    result.push(make(sorted.slice(chunkStart)));
     return result;
 }
 
@@ -279,6 +430,18 @@ export type BaseBulkDatabaseReader = {
     // Each key's value for the column plus the row's write-time. value may be ABSENT (the row never set
     // this column — the join then falls through to an older reader for that key/column).
     getColumn: (column: string) => Promise<{ key: string; value: unknown; time: number }[]>;
+    // Like getColumn but returns each cell's raw on-disk encoding (type tag + bytes) keyed by key, WITHOUT
+    // decoding the value — for merges, which splice the winning bytes straight into the output. ABSENT
+    // cells are omitted (a missing key means this reader didn't set this column, so the merge falls
+    // through to an older reader). Each cell's write-time is the reader's keyTimes value for that key.
+    getRawColumn: (column: string) => Promise<Map<string, RawCell>>;
+    // Returns the column's INDEX (offsets + types) plus a contiguous row-range byte reader. The planned
+    // merge loads this for every (source, column) once — small (~5B/row), no value bytes pulled — and
+    // uses types/offsets to plan the output's byte layout. Execute then reads only the needed runs.
+    getColumnIndex: (column: string) => Promise<ColumnIndex>;
+    // Maps a key to its row index in this reader (and undefined if absent). The planned merge uses this
+    // to look up a winning cell's source row without going through a column read.
+    rowOfKey: (key: string) => number | undefined;
     // The value + write-time for (key, column), or ABSENT if this reader has no such cell.
     getSingleField: (key: string, column: string) => Promise<{ value: unknown; time: number } | typeof ABSENT>;
 };
@@ -352,6 +515,59 @@ export async function loadBulkDatabase(config: {
             const values = await readWholeColumn(column);
             return keys.map((key, i) => ({ key, value: values[i], time: times[i] }));
         },
+        async getRawColumn(column) {
+            const map = new Map<string, RawCell>();
+            const col = colByName.get(column);
+            if (!col) return map; // file lacks this column → every cell ABSENT (fall through)
+            const blob = await config.getRange(dataBase + col.offset, dataBase + col.offset + col.length);
+            const indexSize = 4 * (rowCount + 1) + rowCount;
+            for (let i = 0; i < rowCount; i++) {
+                const type = blob[4 * (rowCount + 1) + i];
+                if (type === TYPE_ABSENT) continue; // omit ABSENT so the join falls through
+                const start = blob.readUInt32LE(4 * i);
+                const end = blob.readUInt32LE(4 * (i + 1));
+                map.set(keys[i], { type, bytes: blob.subarray(indexSize + start, indexSize + end) });
+            }
+            return map;
+        },
+        async getColumnIndex(column) {
+            const col = colByName.get(column);
+            if (!col) {
+                // File lacks this column — present an all-ABSENT index so the planner can treat it
+                // uniformly with files that have the column.
+                const offsets = new Uint32Array(rowCount + 1);
+                const types = new Uint8Array(rowCount).fill(TYPE_ABSENT);
+                return {
+                    offsets,
+                    types,
+                    async readValueBytes() { return EMPTY_BUFFER; },
+                };
+            }
+            const colBase = dataBase + col.offset;
+            const indexSize = 4 * (rowCount + 1) + rowCount;
+            // One read pulls offsets + types (small — ~5 B/row). Block cache makes subsequent value reads
+            // of nearby rows cheap. Decoded into aligned typed arrays so the executor can do O(1) lookups.
+            const indexBuf = await config.getRange(colBase, colBase + indexSize);
+            const offsets = new Uint32Array(rowCount + 1);
+            for (let i = 0; i <= rowCount; i++) offsets[i] = indexBuf.readUInt32LE(4 * i);
+            const types = new Uint8Array(rowCount);
+            for (let i = 0; i < rowCount; i++) types[i] = indexBuf[4 * (rowCount + 1) + i];
+            const dataStart = colBase + indexSize;
+            return {
+                offsets,
+                types,
+                async readValueBytes(startRow, endRow) {
+                    if (endRow <= startRow) return EMPTY_BUFFER;
+                    const start = offsets[startRow];
+                    const end = offsets[endRow];
+                    if (end <= start) return EMPTY_BUFFER;
+                    return config.getRange(dataStart + start, dataStart + end);
+                },
+            };
+        },
+        rowOfKey(key) {
+            return keyIndex.get(key);
+        },
         async getSingleField(key, column) {
             const row = keyIndex.get(key);
             if (row === undefined) return ABSENT;

package/storage/BulkDatabase2/BulkDatabaseMerge.d.ts

@@ -0,0 +1,49 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { BaseBulkDatabaseReader } from "./BulkDatabaseFormat";
+type CopyRun = {
+    sourceIdx: number;
+    sourceStartRow: number;
+    sourceEndRow: number;
+    outputByteStart: number;
+    byteLength: number;
+};
+type PlannedOutputColumn = {
+    name: string;
+    offsets: Uint32Array;
+    types: Uint8Array;
+    dataLength: number;
+    runs: CopyRun[];
+};
+export type PlannedOutputFile = {
+    keys: string[];
+    times: number[];
+    minKey: string;
+    maxKey: string;
+    columns: PlannedOutputColumn[];
+    estimatedFileBytes: number;
+    sourceCounts: Map<number, number>;
+};
+export type PlannedMergeOutput = {
+    name: string;
+    minKey: string;
+    maxKey: string;
+    rowCount: number;
+    size: number;
+    sources: Map<string, number>;
+};
+export declare function runPlannedMerge(config: {
+    sources: BaseBulkDatabaseReader[];
+    sourceNames: string[];
+    collectionName: string;
+    targetFileBytes?: number;
+    targetBatchBytes?: number;
+    writeFile: (data: Buffer) => Promise<{
+        name: string;
+        size: number;
+    }>;
+}): Promise<{
+    outputs: PlannedMergeOutput[];
+    carriedDeletes: Map<string, number>;
+}>;
+export {};