sliftutils 1.7.32 → 1.7.34

package/storage/BulkDatabase2/BulkDatabaseFormat.ts

@@ -1,7 +1,5 @@
 // File format (all integers little-endian):
-//   u32 headerLength, then headerLength bytes of JSON: { rowCount, columns: [{ name, offset, length }] }, where offset is relative to the end of the header.
-//   Each column blob is: u32 offsets[rowCount + 1] (byte offsets into the column's data section), u8 types[rowCount], then the data section (each value's bytes, concatenated).
-// Values are encoded with an explicit type tag per value (see the TYPE_ constants). Reads only fetch the byte ranges they need.
+//   u32 headerLength, then headerLength bytes of JSON: { rowCount, columns: [{ name, offset, length }] }, where offset is relative to the end of the header. Each column blob is: u32 offsets[rowCount + 1] (byte offsets into the column's data section), u8 types[rowCount], then the data section (each value's bytes, concatenated). Values are encoded with an explicit type tag per value (see the TYPE_ constants). Reads only fetch the byte ranges they need.
 
 export const KEY_COLUMN = "key";
 
@@ -20,9 +18,7 @@ const TYPE_INT32_ARRAY = 10;
 const TYPE_UINT32_ARRAY = 11;
 const TYPE_FLOAT32_ARRAY = 12;
 const TYPE_FLOAT64_ARRAY = 13;
-// A cell whose row never set this column at all — as opposed to TYPE_UNDEFINED, an explicitly stored
-// undefined. On read, ABSENT falls through to older readers for that column; a stored undefined stops
-// the fall-through (it's a real value that clears the column).
+// A cell whose row never set this column at all — as opposed to TYPE_UNDEFINED, an explicitly stored undefined. On read, ABSENT falls through to older readers for that column; a stored undefined stops the fall-through (it's a real value that clears the column).
 const TYPE_ABSENT = 14;
 
 const TYPED_ARRAY_TYPES: { type: number; ctor: { new(buffer: ArrayBuffer): ArrayBufferView; BYTES_PER_ELEMENT: number; name: string } }[] = [
@@ -39,54 +35,35 @@ const TYPED_ARRAY_TYPES: { type: number; ctor: { new(buffer: ArrayBuffer): Array
 
 export const EMPTY_BUFFER = Buffer.alloc(0) as Buffer;
 
-// Sentinel a reader returns for a cell whose row never set this column, so the join can fall through
-// to an older reader for that column. Distinct from a stored undefined, which is a real clearing value.
+// Sentinel a reader returns for a cell whose row never set this column, so the join can fall through to an older reader for that column. Distinct from a stored undefined, which is a real clearing value.
 export const ABSENT = Symbol("absent");
 
-// One value exactly as it sits on disk: its type tag + raw bytes. A cell's encoding is position-
-// independent (nothing about it depends on which row/file it lives in), so a merge copies the winning
-// RawCell straight from an input column into the output — never decoding it to a JS value and
-// re-encoding. That's far less memory (no object materialization) and much faster (a byte copy, not a
-// JPEG/typed-array decode + re-encode).
+// One value exactly as it sits on disk: its type tag + raw bytes. A cell's encoding is position- independent (nothing about it depends on which row/file it lives in), so a merge copies the winning RawCell straight from an input column into the output — never decoding it to a JS value and re-encoding. That's far less memory (no object materialization) and much faster (a byte copy, not a JPEG/typed-array decode + re-encode).
 export type RawCell = { type: number; bytes: Buffer };
 
-// The type tag a cell carries when its row never set this column — read it from the column's `types`
-// array to detect ABSENT without decoding. Re-exported so the merge planner (which works at the
-// column-index level, no value materialization) can check it.
+// The type tag a cell carries when its row never set this column — read it from the column's `types` array to detect ABSENT without decoding. Re-exported so the merge planner (which works at the column-index level, no value materialization) can check it.
 export const TYPE_ABSENT_TAG = TYPE_ABSENT;
 
-// A column's index alone (offsets + types — small, ~5 bytes/row) plus a primitive to read a CONTIGUOUS
-// row range's raw value bytes. Used by the planned merge: it loads the index across every (source,
-// column) once (cheap), uses types to detect ABSENT and offsets to size each cell, plans the byte
-// layout of every output file from those — then in the execute phase reads only the runs of bytes it
-// actually needs, copying them straight into pre-laid-out output buffers. No cell value is ever
-// materialized as a JS object.
+// A column's index alone (offsets + types — small, ~5 bytes/row) plus a primitive to read a CONTIGUOUS row range's raw value bytes. Used by the planned merge: it loads the index across every (source, column) once (cheap), uses types to detect ABSENT and offsets to size each cell, plans the byte layout of every output file from those — then in the execute phase reads only the runs of bytes it actually needs, copying them straight into pre-laid-out output buffers. No cell value is ever materialized as a JS object.
 export type ColumnIndex = {
     // offsets[i]..offsets[i+1] is row i's value byte range within the column's data section.
     offsets: Uint32Array;
     // Per-row type tag; TYPE_ABSENT_TAG marks "this row never set this column" (fall-through).
     types: Uint8Array;
-    // Read the contiguous bytes of value(s) for rows [startRow, endRow). Length is exactly
-    // offsets[endRow] - offsets[startRow] — the cells in that range concatenated. The caller composes
-    // larger reads from consecutive rows (one source-side getRange per run).
+    // Read the contiguous bytes of value(s) for rows [startRow, endRow). Length is exactly offsets[endRow] - offsets[startRow] — the cells in that range concatenated. The caller composes larger reads from consecutive rows (one source-side getRange per run).
     readValueBytes: (startRow: number, endRow: number) => Promise<Buffer>;
 };
 
-// Hidden per-row column holding each row's write-time (so reads can resolve a key to its latest value
-// by actual time). NUL-prefixed so it can't collide with a user column; excluded from `columns`.
+// Hidden per-row column holding each row's write-time (so reads can resolve a key to its latest value by actual time). NUL-prefixed so it can't collide with a user column; excluded from `columns`.
 const TIME_COLUMN = String.fromCharCode(0) + "t";
 
 type FileHeader = {
     rowCount: number;
     columns: { name: string; offset: number; length: number }[];
-    // Oldest/newest write-time of the data in this file (from the stream entries it was folded from,
-    // carried through merges). Lets the merge planner pick files overlapping a time range. Absent (0)
-    // in files written before this existed.
+    // Oldest/newest write-time of the data in this file (from the stream entries it was folded from, carried through merges). Lets the merge planner pick files overlapping a time range. Absent (0) in files written before this existed.
     minTime?: number;
     maxTime?: number;
-    // Lexicographically smallest/largest key in this file (rows are stored key-sorted). Lets the merge
-    // planner group/dedup by key range and lets a single-key read skip files whose range excludes the
-    // key. Absent (undefined) in files written before this existed — treated as "spans all keys".
+    // Lexicographically smallest/largest key in this file (rows are stored key-sorted). Lets the merge planner group/dedup by key range and lets a single-key read skip files whose range excludes the key. Absent (undefined) in files written before this existed — treated as "spans all keys".
     minKey?: string;
     maxKey?: string;
 };
@@ -177,9 +154,7 @@ function encodeBulkData(data: unknown[]): Buffer {
     return Buffer.concat([offsets, types, ...parts]);
 }
 
-// Like encodeBulkData but the values are already encoded (a merge supplies the winning cells' raw bytes
-// straight from the input files). Lays out the same offsets / types / data section without touching the
-// values themselves.
+// Like encodeBulkData but the values are already encoded (a merge supplies the winning cells' raw bytes straight from the input files). Lays out the same offsets / types / data section without touching the values themselves.
 function encodeBulkDataRaw(cells: RawCell[]): Buffer {
     const n = cells.length;
     const offsets = Buffer.alloc(4 * (n + 1));
@@ -209,11 +184,7 @@ function decodeBulkData(blob: Buffer, rowCount: number): unknown[] {
     return values;
 }
 
-// Target logical (uncompressed) size of one bulk file. A rows array bigger than this is split across
-// multiple key-contiguous files, so files stay around this size (the merge policy's target) and no
-// single column blob / Buffer.concat ever approaches the ~2GB Buffer length limit. The estimate is
-// rough on purpose — it only needs to keep each chunk near the target, not be exact. A single row
-// bigger than the target still becomes its own (oversized) file, since we never split within a key.
+// Target logical (uncompressed) size of one bulk file. A rows array bigger than this is split across multiple key-contiguous files, so files stay around this size (the merge policy's target) and no single column blob / Buffer.concat ever approaches the ~2GB Buffer length limit. The estimate is rough on purpose — it only needs to keep each chunk near the target, not be exact. A single row bigger than the target still becomes its own (oversized) file, since we never split within a key.
 export const TARGET_FILE_BYTES = 256 * 1024 * 1024;
 
 // Per-value on-disk overhead: a 4-byte offset entry plus a 1-byte type tag.
@@ -237,9 +208,7 @@ function estimateRowBytes(row: Record<string, unknown>): number {
     return total;
 }
 
-// Concatenates already-encoded column blobs into a complete file (4-byte header length + header JSON +
-// blobs), computing the header's time + key bounds from the per-row times and keys. Shared by the
-// object-based builder (buildOneFile) and the raw-splice builder (buildOneFileRaw).
+// Concatenates already-encoded column blobs into a complete file (4-byte header length + header JSON + blobs), computing the header's time + key bounds from the per-row times and keys. Shared by the object-based builder (buildOneFile) and the raw-splice builder (buildOneFileRaw).
 function assembleFile(columnNames: string[], blobs: Buffer[], rowCount: number, times: number[], keys: string[]): Buffer {
     let offset = 0;
     const columns = columnNames.map((name, i) => {
@@ -281,21 +250,15 @@ function buildOneFile(rows: Record<string, unknown>[], times: number[]): Buffer
     return assembleFile(columnNames, blobs, rows.length, times, rows.map(r => r[KEY_COLUMN] as string));
 }
 
-// A resolved output row for the raw-splice merge: its key, write-time, and the winning raw cell for each
-// column it has (a column it lacks is written ABSENT — fall-through). The cell bytes are copied straight
-// from the input files; no value is ever decoded.
+// A resolved output row for the raw-splice merge: its key, write-time, and the winning raw cell for each column it has (a column it lacks is written ABSENT — fall-through). The cell bytes are copied straight from the input files; no value is ever decoded.
 export type RawRow = { key: string; time: number; cells: Map<string, RawCell> };
 
-// Size of a column blob's INDEX section (offsets array + types array) for a column of N rows. The data
-// section follows immediately after. The planned merge uses this to size output buffers exactly.
+// Size of a column blob's INDEX section (offsets array + types array) for a column of N rows. The data section follows immediately after. The planned merge uses this to size output buffers exactly.
 export function columnIndexByteLength(rowCount: number): number {
     return 4 * (rowCount + 1) + rowCount;
 }
 
-// Assembles a complete bulk file from a set of pre-built value column blobs plus the keys + times. Auto-
-// adds the KEY_COLUMN (keys) and hidden TIME_COLUMN (times) — they're small enough to encode in one shot
-// here. Used by the planned merge: it builds each value column's blob by-hand (offsets/types + raw bytes
-// copied directly from inputs, no value materialization) and hands the result here for header + bounds.
+// Assembles a complete bulk file from a set of pre-built value column blobs plus the keys + times. Auto- adds the KEY_COLUMN (keys) and hidden TIME_COLUMN (times) — they're small enough to encode in one shot here. Used by the planned merge: it builds each value column's blob by-hand (offsets/types + raw bytes copied directly from inputs, no value materialization) and hands the result here for header + bounds.
 export function assemblePlannedFile(config: {
     valueColumns: { name: string; blob: Buffer }[];
     keys: string[];
@@ -313,8 +276,7 @@ export function assemblePlannedFile(config: {
 const ABSENT_CELL: RawCell = { type: TYPE_ABSENT, bytes: EMPTY_BUFFER };
 
 function buildOneFileRaw(rows: RawRow[]): Buffer {
-    // Columns present in this chunk, in first-seen order (matches buildOneFile, which only emits columns
-    // some row actually has).
+    // Columns present in this chunk, in first-seen order (matches buildOneFile, which only emits columns some row actually has).
     const valueColumns: string[] = [];
     const seen = new Set<string>();
     for (const row of rows) for (const col of row.cells.keys()) {
@@ -338,16 +300,10 @@ function estimateRawRowBytes(row: RawRow): number {
     return total;
 }
 
-// One complete, independent file: the encoded buffer plus its key range + row count (the caller logs the
-// range when a merge splits across several files).
+// One complete, independent file: the encoded buffer plus its key range + row count (the caller logs the range when a merge splits across several files).
 export interface BuiltFile { buffer: Buffer; minKey: string; maxKey: string; rowCount: number; }
 
-// Returns one complete, independent file per chunk of rows. Rows are first sorted by key, then
-// partitioned into key-contiguous chunks of ~targetBytes each — so every returned file is key-sorted
-// (tight minKey/maxKey for the read-skip + merge planner) and stays near the target size, and no
-// single column blob / Buffer.concat approaches the ~2GB limit. The chunks have disjoint key ranges,
-// so the caller just writes each as its own file. A normal-sized write returns a single file.
-// `times[i]` is row i's write-time, stored per row so reads resolve a key to its latest value by time.
+// Returns one complete, independent file per chunk of rows. Rows are first sorted by key, then partitioned into key-contiguous chunks of ~targetBytes each — so every returned file is key-sorted (tight minKey/maxKey for the read-skip + merge planner) and stays near the target size, and no single column blob / Buffer.concat approaches the ~2GB limit. The chunks have disjoint key ranges, so the caller just writes each as its own file. A normal-sized write returns a single file. `times[i]` is row i's write-time, stored per row so reads resolve a key to its latest value by time.
 export function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes = TARGET_FILE_BYTES): BuiltFile[] {
     // A chunk is already key-sorted, so its first/last row are its min/max key.
     const make = (rs: Record<string, unknown>[], ts: number[]): BuiltFile => ({
@@ -381,9 +337,7 @@ export function buildFileBuffer(rows: Record<string, unknown>[], times: number[]
     return result;
 }
 
-// The raw-splice counterpart of buildFileBuffer, used by merges: the rows already carry their winning
-// cells as raw on-disk bytes (no JS values), so this just sorts by key, chunks to ~targetBytes, and
-// concatenates the bytes. Same output guarantees: key-contiguous, disjoint, ascending files.
+// The raw-splice counterpart of buildFileBuffer, used by merges: the rows already carry their winning cells as raw on-disk bytes (no JS values), so this just sorts by key, chunks to ~targetBytes, and concatenates the bytes. Same output guarantees: key-contiguous, disjoint, ascending files.
 export function buildFileBufferRaw(rows: RawRow[], targetBytes = TARGET_FILE_BYTES): BuiltFile[] {
     const make = (rs: RawRow[]): BuiltFile => ({
         buffer: buildOneFileRaw(rs),
@@ -410,8 +364,7 @@ export function buildFileBufferRaw(rows: RawRow[], targetBytes = TARGET_FILE_BYT
 }
 
 export type BaseBulkDatabaseReader = {
-    // Identifies the source this reader came from (the bulk file name, or "(streams)") so the join can
-    // name the offending file when one of its reads fails. Undefined for readers built without one.
+    // Identifies the source this reader came from (the bulk file name, or "(streams)") so the join can name the offending file when one of its reads fails. Undefined for readers built without one.
     name?: string;
     rowCount: number;
     totalBytes: number;
@@ -424,33 +377,23 @@ export type BaseBulkDatabaseReader = {
     // Keys is special, it's always automatically decoded, even though it is stored as a normal column
     keys: string[];
     columns: { column: string; byteSize: number }[];
-    // Each key's row write-time (the time of its newest write in this reader). The join compares these
-    // across readers to resolve a key to its latest value.
+    // Each key's row write-time (the time of its newest write in this reader). The join compares these across readers to resolve a key to its latest value.
     keyTimes: Map<string, number>;
-    // Per-key tombstone time: the key was deleted at this time. The join treats a delete like any other
-    // event — a delete only wins if it's newer than every set for the key. Only the stream reader sets it.
+    // Per-key tombstone time: the key was deleted at this time. The join treats a delete like any other event — a delete only wins if it's newer than every set for the key. Only the stream reader sets it.
     deleteTimes?: Map<string, number>;
-    // Each key's value for the column plus the row's write-time. value may be ABSENT (the row never set
-    // this column — the join then falls through to an older reader for that key/column).
+    // Each key's value for the column plus the row's write-time. value may be ABSENT (the row never set this column — the join then falls through to an older reader for that key/column).
     getColumn: (column: string) => Promise<{ key: string; value: unknown; time: number }[]>;
-    // Like getColumn but returns each cell's raw on-disk encoding (type tag + bytes) keyed by key, WITHOUT
-    // decoding the value — for merges, which splice the winning bytes straight into the output. ABSENT
-    // cells are omitted (a missing key means this reader didn't set this column, so the merge falls
-    // through to an older reader). Each cell's write-time is the reader's keyTimes value for that key.
+    // Like getColumn but returns each cell's raw on-disk encoding (type tag + bytes) keyed by key, WITHOUT decoding the value — for merges, which splice the winning bytes straight into the output. ABSENT cells are omitted (a missing key means this reader didn't set this column, so the merge falls through to an older reader). Each cell's write-time is the reader's keyTimes value for that key.
     getRawColumn: (column: string) => Promise<Map<string, RawCell>>;
-    // Returns the column's INDEX (offsets + types) plus a contiguous row-range byte reader. The planned
-    // merge loads this for every (source, column) once — small (~5B/row), no value bytes pulled — and
-    // uses types/offsets to plan the output's byte layout. Execute then reads only the needed runs.
+    // Returns the column's INDEX (offsets + types) plus a contiguous row-range byte reader. The planned merge loads this for every (source, column) once — small (~5B/row), no value bytes pulled — and uses types/offsets to plan the output's byte layout. Execute then reads only the needed runs.
     getColumnIndex: (column: string) => Promise<ColumnIndex>;
-    // Maps a key to its row index in this reader (and undefined if absent). The planned merge uses this
-    // to look up a winning cell's source row without going through a column read.
+    // Maps a key to its row index in this reader (and undefined if absent). The planned merge uses this to look up a winning cell's source row without going through a column read.
     rowOfKey: (key: string) => number | undefined;
     // The value + write-time for (key, column), or ABSENT if this reader has no such cell.
     getSingleField: (key: string, column: string) => Promise<{ value: unknown; time: number } | typeof ABSENT>;
 };
 
-// Reads just the file header (the 4-byte length + header JSON) — no column data. Used by the merge
-// planner to get each file's row count, time range, and key range cheaply across many files.
+// Reads just the file header (the 4-byte length + header JSON) — no column data. Used by the merge planner to get each file's row count, time range, and key range cheaply across many files.
 export type BulkHeaderInfo = { rowCount: number; minTime: number; maxTime: number; minKey?: string; maxKey?: string; columns: { column: string; byteSize: number }[] };
 export async function loadBulkHeader(getRange: (start: number, end: number) => Promise<Buffer>, totalBytes: number): Promise<BulkHeaderInfo> {
     const headerLength = (await getRange(0, 4)).readUInt32LE(0);
@@ -499,8 +442,7 @@ export async function loadBulkDatabase(config: {
     });
     const keyIndex = new Map(keys.map((key, i) => [key, i]));
 
-    // Per-row write-times. Old files (written before this column existed) fall back to the file's header
-    // time (or 0) for every row — fine, since such files predate concurrent-time resolution.
+    // Per-row write-times. Old files (written before this column existed) fall back to the file's header time (or 0) for every row — fine, since such files predate concurrent-time resolution.
     const times: number[] = colByName.has(TIME_COLUMN)
         ? (await readWholeColumn(TIME_COLUMN)).map(v => typeof v === "number" ? v : 0)
         : keys.map(() => header.maxTime || 0);
@@ -538,8 +480,7 @@ export async function loadBulkDatabase(config: {
         async getColumnIndex(column) {
             const col = colByName.get(column);
             if (!col) {
-                // File lacks this column — present an all-ABSENT index so the planner can treat it
-                // uniformly with files that have the column.
+                // File lacks this column — present an all-ABSENT index so the planner can treat it uniformly with files that have the column.
                 const offsets = new Uint32Array(rowCount + 1);
                 const types = new Uint8Array(rowCount).fill(TYPE_ABSENT);
                 return {
@@ -550,8 +491,7 @@ export async function loadBulkDatabase(config: {
             }
             const colBase = dataBase + col.offset;
             const indexSize = 4 * (rowCount + 1) + rowCount;
-            // One read pulls offsets + types (small — ~5 B/row). Block cache makes subsequent value reads
-            // of nearby rows cheap. Decoded into aligned typed arrays so the executor can do O(1) lookups.
+            // One read pulls offsets + types (small — ~5 B/row). Block cache makes subsequent value reads of nearby rows cheap. Decoded into aligned typed arrays so the executor can do O(1) lookups.
             const indexBuf = await config.getRange(colBase, colBase + indexSize);
             const offsets = new Uint32Array(rowCount + 1);
             for (let i = 0; i <= rowCount; i++) offsets[i] = indexBuf.readUInt32LE(4 * i);

package/storage/BulkDatabase2/BulkDatabaseMerge.ts

@@ -10,13 +10,10 @@ import {
     TYPE_ABSENT_TAG,
 } from "./BulkDatabaseFormat";
 
-// Default total in-flight buffer budget for the executor: how much output value-data we hold in memory at
-// once. Multiple output files (≤ TARGET_FILE_BYTES each) are built together inside this budget so reading
-// inputs once amortizes their I/O across several outputs. Tighter budget → more passes, less memory.
+// Default total in-flight buffer budget for the executor: how much output value-data we hold in memory at once. Multiple output files (≤ TARGET_FILE_BYTES each) are built together inside this budget so reading inputs once amortizes their I/O across several outputs. Tighter budget → more passes, less memory.
 const DEFAULT_OUTPUT_BATCH_BYTES = 2 * 1024 * 1024 * 1024;
 
-// Per-value on-disk overhead (4-byte offset entry + 1-byte type tag). Used for the partition-cut size
-// estimate; the executor doesn't use it (offsets/types are sized exactly from the plan).
+// Per-value on-disk overhead (4-byte offset entry + 1-byte type tag). Used for the partition-cut size estimate; the executor doesn't use it (offsets/types are sized exactly from the plan).
 const PER_VALUE_OVERHEAD = 5;
 
 function fmtBytes(n: number): string {
@@ -26,9 +23,7 @@ function fmtBytes(n: number): string {
     return (n / 1024 / 1024 / 1024).toFixed(2) + "GB";
 }
 
-// A contiguous copy operation: take `sourceEndRow - sourceStartRow` consecutive rows of one source
-// column, splice the resulting bytes into the output column's data section at outputByteStart. Adjacent
-// per-row decisions get RLE'd into one copy op — one source-side read + memcpy instead of one per row.
+// A contiguous copy operation: take `sourceEndRow - sourceStartRow` consecutive rows of one source column, splice the resulting bytes into the output column's data section at outputByteStart. Adjacent per-row decisions get RLE'd into one copy op — one source-side read + memcpy instead of one per row.
 type CopyRun = {
     sourceIdx: number;
     sourceStartRow: number;
@@ -39,8 +34,7 @@ type CopyRun = {
 
 type PlannedOutputColumn = {
     name: string;
-    // offsets[i]..offsets[i+1] = row i's value byte range in the column's data section. Populated by the
-    // planner; the executor writes it verbatim into the output column blob.
+    // offsets[i]..offsets[i+1] = row i's value byte range in the column's data section. Populated by the planner; the executor writes it verbatim into the output column blob.
     offsets: Uint32Array;
     // Per-row type tag (TYPE_ABSENT for fall-through). Populated by the planner.
     types: Uint8Array;
@@ -71,17 +65,7 @@ export type PlannedMergeOutput = {
     sources: Map<string, number>;
 };
 
-// Plan a merge over `sources` and execute it, writing one or more output files via the caller's
-// `writeFile`. Two phases:
-//   • Planning: load only per-column INDEXES (offsets + types — ~5 B/row, small even on 20GB
-//     collections), determine the winning cell per (live key, column) by newest write-time + non-ABSENT
-//     fall-through, sort keys, partition into ~targetFileBytes output files, and pre-compute each output
-//     column's offsets/types arrays plus a run-length-encoded copy plan (no value data touched yet).
-//   • Execute: group outputs into ~targetBatchBytes batches; per batch, allocate column blobs with
-//     offsets/types already filled in, then iterate input sources copying contiguous byte runs straight
-//     into the output buffers. A source with no contribution to this batch is skipped entirely.
-// Returns the new output file descriptors plus any tombstones whose newest event is still a delete
-// (carried forward when older files outside the merge could still hold a now-stale set for that key).
+// Plan a merge over `sources` and execute it, writing one or more output files via the caller's `writeFile`. Two phases: • Planning: load only per-column INDEXES (offsets + types — ~5 B/row, small even on 20GB collections), determine the winning cell per (live key, column) by newest write-time + non-ABSENT fall-through, sort keys, partition into ~targetFileBytes output files, and pre-compute each output column's offsets/types arrays plus a run-length-encoded copy plan (no value data touched yet). • Execute: group outputs into ~targetBatchBytes batches; per batch, allocate column blobs with offsets/types already filled in, then iterate input sources copying contiguous byte runs straight into the output buffers. A source with no contribution to this batch is skipped entirely. Returns the new output file descriptors plus any tombstones whose newest event is still a delete (carried forward when older files outside the merge could still hold a now-stale set for that key).
 export async function runPlannedMerge(config: {
     sources: BaseBulkDatabaseReader[];
     sourceNames: string[];
@@ -97,11 +81,7 @@ export async function runPlannedMerge(config: {
     const targetBatchBytes = config.targetBatchBytes ?? DEFAULT_OUTPUT_BATCH_BYTES;
     const log = config.log ?? (line => console.log(`${blue(config.collectionName)} ${line}`));
 
-    // ─────────────────────────────────────────── Phase 1: plan ───────────────────────────────────────────
-    // Load per-source column indexes FIRST so a file that's gone (deleted between caller's listFiles and
-    // now) gets caught here and excluded from the merge entirely. Each source's load is wrapped in a
-    // try/catch; on failure we mark the source bad, log a "skipped" line, and the rest of planning +
-    // execute pretends the source doesn't exist. The caller only deletes inputs that were actually used.
+    // ─────────────────────────────────────────── Phase 1: plan ─────────────────────────────────────────── Load per-source column indexes FIRST so a file that's gone (deleted between caller's listFiles and now) gets caught here and excluded from the merge entirely. Each source's load is wrapped in a try/catch; on failure we mark the source bad, log a "skipped" line, and the rest of planning + execute pretends the source doesn't exist. The caller only deletes inputs that were actually used.
     const sourceOk = new Array<boolean>(config.sources.length).fill(true);
     const indexesPerSource: Map<string, ColumnIndex>[] = await Promise.all(config.sources.map(async (src, si) => {
         const map = new Map<string, ColumnIndex>();
@@ -112,8 +92,7 @@ export async function runPlannedMerge(config: {
         } catch (e) {
             sourceOk[si] = false;
             log(`${magenta("skipped")} ${config.sourceNames[si]}: ${(e as Error).message}`);
-            // Stack goes straight to console (not the buffered step log) so the merge summary stays
-            // readable but we can still trace where the failure came from.
+            // Stack goes straight to console (not the buffered step log) so the merge summary stays readable but we can still trace where the failure came from.
             console.log(`${config.collectionName} skipped ${config.sourceNames[si]}:`, (e as Error).stack || e);
         }
         return map;
@@ -123,9 +102,7 @@ export async function runPlannedMerge(config: {
         if (sourceOk[si]) usedSourceNames.add(config.sourceNames[si]);
     }
 
-    // Aggregate keyTimes + deleteTimes across GOOD sources only (max per key). A bad source's cached
-    // metadata might claim a key as newest, but we can't actually read its cells — letting it win the
-    // time resolution would just drop that key out of the output.
+    // Aggregate keyTimes + deleteTimes across GOOD sources only (max per key). A bad source's cached metadata might claim a key as newest, but we can't actually read its cells — letting it win the time resolution would just drop that key out of the output.
     const deleteTime = new Map<string, number>();
     const keyTime = new Map<string, number>();
     for (let si = 0; si < config.sources.length; si++) {
@@ -143,8 +120,7 @@ export async function runPlannedMerge(config: {
         }
     }
 
-    // Live keys (newest set strictly newer than newest delete) + tombstones to carry forward (newest
-    // event is a delete and the merge doesn't include the oldest data — that's the caller's call).
+    // Live keys (newest set strictly newer than newest delete) + tombstones to carry forward (newest event is a delete and the merge doesn't include the oldest data — that's the caller's call).
     const liveKeys: string[] = [];
     for (const [k, t] of keyTime) {
         const dT = deleteTime.get(k) ?? -Infinity;
@@ -156,8 +132,7 @@ export async function runPlannedMerge(config: {
         if (dT >= sT) carriedDeletes.set(k, dT);
     }
 
-    // All distinct value columns from GOOD sources (KEY_COLUMN excluded — it's added at file-assembly
-    // time). Order: first-seen, matching the existing builders.
+    // All distinct value columns from GOOD sources (KEY_COLUMN excluded — it's added at file-assembly time). Order: first-seen, matching the existing builders.
     const allColumns: string[] = [];
     const seenCols = new Set<string>();
     for (let si = 0; si < config.sources.length; si++) {
@@ -169,9 +144,7 @@ export async function runPlannedMerge(config: {
         }
     }
 
-    // For each live key, for each column, find the winning source: among sources that have this key, the
-    // one whose keyTime is largest AND whose column-index reports non-ABSENT. Record source + sourceRow +
-    // byteLen + type so the planner can size offsets and the executor can copy the bytes.
+    // For each live key, for each column, find the winning source: among sources that have this key, the one whose keyTime is largest AND whose column-index reports non-ABSENT. Record source + sourceRow + byteLen + type so the planner can size offsets and the executor can copy the bytes.
     type CellChoice = { sourceIdx: number; sourceRow: number; byteLen: number; type: number };
     const cellsPerKey = new Map<string, (CellChoice | undefined)[]>();
     for (const key of liveKeys) {
@@ -206,12 +179,10 @@ export async function runPlannedMerge(config: {
         cellsPerKey.set(key, cells);
     }
 
-    // Sort live keys lexicographically so each output file is key-contiguous (tight minKey/maxKey + fast
-    // single-key reads via header skip).
+    // Sort live keys lexicographically so each output file is key-contiguous (tight minKey/maxKey + fast single-key reads via header skip).
     liveKeys.sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
 
-    // Per-key estimated bytes for partition cutting: KEY_COLUMN cell + per value-column cell (bytes + 5B
-    // overhead) + TIME_COLUMN cell. Rough — only needs to keep each output near the target.
+    // Per-key estimated bytes for partition cutting: KEY_COLUMN cell + per value-column cell (bytes + 5B overhead) + TIME_COLUMN cell. Rough — only needs to keep each output near the target.
     function keyTotalBytes(key: string): number {
         const cells = cellsPerKey.get(key)!;
         let total = key.length * 2 + PER_VALUE_OVERHEAD;
@@ -220,8 +191,7 @@ export async function runPlannedMerge(config: {
         return total;
     }
 
-    // Walk sorted keys, cut each file when accumulated bytes would exceed targetFileBytes. A single key
-    // larger than the target still becomes its own (oversized) file — we never split within a key.
+    // Walk sorted keys, cut each file when accumulated bytes would exceed targetFileBytes. A single key larger than the target still becomes its own (oversized) file — we never split within a key.
     const fileKeyRanges: { start: number; end: number }[] = [];
     {
         let chunkStart = 0;
@@ -243,9 +213,7 @@ export async function runPlannedMerge(config: {
 
     log(`${magenta("plan")}: ${formatNumber(liveKeys.length)} live keys, ${plans.length} output file(s), ${formatNumber(carriedDeletes.size)} tombstones carried`);
 
-    // ───────────────────────────────────────── Phase 2: execute ──────────────────────────────────────────
-    // Group output files into batches that fit within targetBatchBytes so we read inputs once per batch
-    // (and skip inputs that contribute nothing to this batch).
+    // ───────────────────────────────────────── Phase 2: execute ────────────────────────────────────────── Group output files into batches that fit within targetBatchBytes so we read inputs once per batch (and skip inputs that contribute nothing to this batch).
     const batches: PlannedOutputFile[][] = [];
     {
         let start = 0;
@@ -305,9 +273,7 @@ function buildOutputPlan(
             }
             types[i] = cell.type;
             sourceCounts.set(cell.sourceIdx, (sourceCounts.get(cell.sourceIdx) ?? 0) + 1);
-            // RLE: extend the current run iff the next cell is the next row of the same source (its
-            // source-side bytes sit immediately after, so one read suffices) AND non-ABSENT (we never
-            // landed in the break-path above).
+            // RLE: extend the current run iff the next cell is the next row of the same source (its source-side bytes sit immediately after, so one read suffices) AND non-ABSENT (we never landed in the break-path above).
             if (currentRun && currentRun.sourceIdx === cell.sourceIdx && currentRun.sourceEndRow === cell.sourceRow) {
                 currentRun.sourceEndRow = cell.sourceRow + 1;
                 currentRun.byteLength += cell.byteLen;
@@ -329,8 +295,7 @@ function buildOutputPlan(
         return { name: colName, offsets, types, dataLength: outputByte, runs };
     });
 
-    // File size estimate: KEY_COLUMN blob + value column blobs + TIME_COLUMN blob + header guess. Used to
-    // batch output files; only needs to be close, not exact.
+    // File size estimate: KEY_COLUMN blob + value column blobs + TIME_COLUMN blob + header guess. Used to batch output files; only needs to be close, not exact.
     let estimatedFileBytes = 4 + 2048;
     let keyBytes = 0;
     for (const k of keys) keyBytes += k.length * 2;
@@ -357,8 +322,7 @@ async function executeBatch(
     writeFile: (data: Buffer) => Promise<{ name: string; size: number }>,
     log: (line: string) => void,
 ): Promise<PlannedMergeOutput[]> {
-    // Allocate one big buffer per (output file, column) — offsets + types are written immediately from
-    // the plan, the data section is filled below by the per-source copy loop.
+    // Allocate one big buffer per (output file, column) — offsets + types are written immediately from the plan, the data section is filled below by the per-source copy loop.
     type ColumnBuffer = { name: string; blob: Buffer; dataStart: number };
     const blobsPerFile: ColumnBuffer[][] = plans.map(plan => plan.columns.map(col => {
         const indexSize = columnIndexByteLength(col.offsets.length - 1);
@@ -368,9 +332,7 @@ async function executeBatch(
         return { name: col.name, blob, dataStart: indexSize };
     }));
 
-    // For each source, copy its byte runs into every output column whose plan references it. A source
-    // that's not referenced by any column in this batch is skipped entirely — that's the point of the
-    // batch shape: read 20GB of inputs only once for the whole pass, not once per output.
+    // For each source, copy its byte runs into every output column whose plan references it. A source that's not referenced by any column in this batch is skipped entirely — that's the point of the batch shape: read 20GB of inputs only once for the whole pass, not once per output.
     for (let si = 0; si < sources.length; si++) {
         let contributes = false;
         for (let fi = 0; fi < plans.length && !contributes; fi++) {

package/storage/BulkDatabase2/BulkDatabaseReader.ts

@@ -21,10 +21,7 @@ export type ReaderConfig = {
     maxTriggerThrottleMs?: number;
 };
 
-// Owns the current LoadedIndex and WriteOverlay and serves every read (async + sync reactive). Apply
-// writes/deletes through applyWrite/applyDelete so the right signals fire. The host swaps in a fresh
-// LoadedIndex atomically via setIndex once the new one has fully built — no lazy rebuild on the next
-// read, no synchronous lag spike.
+// Owns the current LoadedIndex and WriteOverlay and serves every read (async + sync reactive). Apply writes/deletes through applyWrite/applyDelete so the right signals fire. The host swaps in a fresh LoadedIndex atomically via setIndex once the new one has fully built — no lazy rebuild on the next read, no synchronous lag spike.
 export class BulkDatabaseReader<T extends { key: string }> {
     constructor(private readonly cfg: ReaderConfig) { }
 
@@ -89,8 +86,7 @@ export class BulkDatabaseReader<T extends { key: string }> {
         return -Infinity;
     }
 
-    // Counter (so concurrent / nested merges in the same instance compose correctly). Signal fires
-    // immediately, not through the trigger throttle — UI spinners should show right away.
+    // Counter (so concurrent / nested merges in the same instance compose correctly). Signal fires immediately, not through the trigger throttle — UI spinners should show right away.
     private compactingCount = 0;
     beginCompaction(): void {
         this.compactingCount++;
@@ -247,9 +243,7 @@ export class BulkDatabaseReader<T extends { key: string }> {
         return false;
     }
 
-    // ── helpers ──────────────────────────────────────────────────────────────────────────────────────
-    // Both async reads await this. The host wires an `ensureIndex` that triggers the initial build if
-    // no index is loaded yet — reads that race the first load all wait on the same promise.
+    // ── helpers ────────────────────────────────────────────────────────────────────────────────────── Both async reads await this. The host wires an `ensureIndex` that triggers the initial build if no index is loaded yet — reads that race the first load all wait on the same promise.
     setEnsureIndex(fn: () => Promise<LoadedIndex<T>>): void { this.ensureIndexFn = fn; }
     private ensureIndexFn: (() => Promise<LoadedIndex<T>>) | undefined;
     private async requireIndex(): Promise<LoadedIndex<T>> {
@@ -262,9 +256,7 @@ export class BulkDatabaseReader<T extends { key: string }> {
         return `(collection has ${blue(formatNumber(index.reader.rowCount))} rows, ${blue(formatNumber(index.reader.totalBytes))}B)`;
     }
 
-    // Notifications are rampingly delayed under sustained changes (so a high-rate writer can't re-run
-    // watchers per change). Underlying data is always current — only the observable notification is
-    // batched/delayed.
+    // Notifications are rampingly delayed under sustained changes (so a high-rate writer can't re-run watchers per change). Underlying data is always current — only the observable notification is batched/delayed.
     private invalidateSignal(signal: string): void {
         const maxMs = this.cfg.maxTriggerThrottleMs ?? 0;
         if (maxMs <= 0) { this.cfg.deps.invalidate(signal); return; }

package/storage/BulkDatabase2/LoadedIndex.ts

@@ -39,10 +39,7 @@ function nullJoin(a: string, b: string): string {
     return a + String.fromCharCode(0) + b;
 }
 
-// A loaded snapshot of one collection's on-disk state: the resolved reader + the per-file decoded sub-
-// readers it stitched, the stream file sizes/times it observed, and the sync (reactive) caches that
-// surface column/field reads. Built once via LoadedIndex.build — eagerly, so the swap into a fresh
-// LoadedIndex never blocks a read on a synchronous rebuild.
+// A loaded snapshot of one collection's on-disk state: the resolved reader + the per-file decoded sub- readers it stitched, the stream file sizes/times it observed, and the sync (reactive) caches that surface column/field reads. Built once via LoadedIndex.build — eagerly, so the swap into a fresh LoadedIndex never blocks a read on a synchronous rebuild.
 export class LoadedIndex<T extends { key: string }> {
     private constructor(
         public readonly name: string,
@@ -67,8 +64,7 @@ export class LoadedIndex<T extends { key: string }> {
     private baseColumnsLoading = new Set<string>();
     private baseFields = new Map<string, { value: unknown; time: number } | undefined>();
     private baseFieldsLoading = new Set<string>();
-    // Last-known values from the previous index, served until this index loads its own fresh value, so
-    // a swap doesn't flash empty in sync reads. Cleared per-entry when the fresh value lands.
+    // Last-known values from the previous index, served until this index loads its own fresh value, so a swap doesn't flash empty in sync reads. Cleared per-entry when the fresh value lands.
     private staleBaseColumns = new Map<string, { key: string; value: unknown; time: number }[]>();
     private staleBaseFields = new Map<string, { value: unknown; time: number } | undefined>();
 
@@ -150,9 +146,7 @@ export class LoadedIndex<T extends { key: string }> {
         }
     }
 
-    // Adopts the previous index's loaded base values as STALE on this one, so a sync read mid-swap can
-    // serve the old value (patched with the current overlay) instead of flashing empty while the fresh
-    // load runs. Per-entry: dropped when the fresh value lands here.
+    // Adopts the previous index's loaded base values as STALE on this one, so a sync read mid-swap can serve the old value (patched with the current overlay) instead of flashing empty while the fresh load runs. Per-entry: dropped when the fresh value lands here.
     inheritStaleFrom(prev: LoadedIndex<T>): void {
         for (const [k, v] of prev.baseColumns) this.staleBaseColumns.set(k, v);
         for (const [k, v] of prev.staleBaseColumns) if (!this.staleBaseColumns.has(k)) this.staleBaseColumns.set(k, v);
@@ -203,9 +197,7 @@ export class LoadedIndex<T extends { key: string }> {
         })();
     }
 
-    // Returns the loaded value (fresh if available, else last-known stale) for the column, or undefined
-    // if neither is present yet. The boolean indicates whether the returned value is fresh; callers may
-    // skip caching a stale value.
+    // Returns the loaded value (fresh if available, else last-known stale) for the column, or undefined if neither is present yet. The boolean indicates whether the returned value is fresh; callers may skip caching a stale value.
     getBaseColumn(column: string): { entries: { key: string; value: unknown; time: number }[]; fresh: boolean } | undefined {
         const fresh = this.baseColumns.get(column);
         if (fresh) return { entries: fresh, fresh: true };
@@ -230,8 +222,7 @@ export class LoadedIndex<T extends { key: string }> {
         return this.baseFields.has(ck) || this.staleBaseFields.has(ck);
     }
 
-    // Drops in-memory loaded values for this index, so a subsequent read reloads fresh from disk. Used
-    // by reloadFromDisk() — a hard reset, no stale fallback survives.
+    // Drops in-memory loaded values for this index, so a subsequent read reloads fresh from disk. Used by reloadFromDisk() — a hard reset, no stale fallback survives.
     dropLoadedValues(): void {
         this.baseColumns.clear();
         this.baseColumnsLoading.clear();
@@ -330,10 +321,7 @@ function pruneSubCaches(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[]
     for (const n of subCaches.stream.keys()) if (!liveStream.has(n)) subCaches.stream.delete(n);
 }
 
-// A corrupt/unreadable column in ONE underlying file must not break the whole joined read. When a
-// source's column read throws, we drop that source for that column (falling through to older readers,
-// exactly as if the file never set the column) and warn once per (file, column) so the corruption is
-// visible without flooding the log on every rebuild.
+// A corrupt/unreadable column in ONE underlying file must not break the whole joined read. When a source's column read throws, we drop that source for that column (falling through to older readers, exactly as if the file never set the column) and warn once per (file, column) so the corruption is visible without flooding the log on every rebuild.
 const warnedCorruptReads = new Set<string>();
 function warnCorruptRead(db: BaseBulkDatabaseReader, column: string, e: unknown): void {
     const path = db.name || "(unknown source)";

package/storage/BulkDatabase2/WriteOverlay.ts

@@ -2,10 +2,7 @@ export const DELETED = Symbol("deleted");
 
 export type OverlayEntry = { time: number; value: Record<string, unknown> | typeof DELETED };
 
-// In-memory pending mutations layered on top of a LoadedIndex. Each entry carries the write's unique
-// timestamp so cross-tab writes can be ordered + the swap can drop entries the new index already
-// covers. Methods that mutate report which columns invalidate ("all" if the key's liveness flipped),
-// so the host can drop the right caches without scanning every column.
+// In-memory pending mutations layered on top of a LoadedIndex. Each entry carries the write's unique timestamp so cross-tab writes can be ordered + the swap can drop entries the new index already covers. Methods that mutate report which columns invalidate ("all" if the key's liveness flipped), so the host can drop the right caches without scanning every column.
 export class WriteOverlay {
     private entries = new Map<string, OverlayEntry>();
 
@@ -29,16 +26,14 @@ export class WriteOverlay {
 
     clear(): void { this.entries.clear(); }
 
-    // Drop entries the new index already reflects (its keyTime or deleteTime ≥ the entry's time). The
-    // remaining entries are writes the index doesn't yet see — they have to keep serving reads.
+    // Drop entries the new index already reflects (its keyTime or deleteTime ≥ the entry's time). The remaining entries are writes the index doesn't yet see — they have to keep serving reads.
     sweepCovered(authority: (key: string) => number): void {
         for (const [key, entry] of this.entries) {
             if (entry.time <= authority(key)) this.entries.delete(key);
         }
     }
 
-    // Apply this overlay onto the base column entries. A column-unset overlay entry leaves the base
-    // value+time in place (a partial write only overrides the columns it set).
+    // Apply this overlay onto the base column entries. A column-unset overlay entry leaves the base value+time in place (a partial write only overrides the columns it set).
     patchColumn(base: { key: string; value: unknown; time: number }[], column: string): { key: string; value: unknown; time: number }[] {
         if (this.entries.size === 0) return base;
         const map = new Map(base.map(e => [e.key, { value: e.value, time: e.time }]));