sliftutils 1.3.1 → 1.4.0

package/storage/BulkDatabase2/BulkDatabaseFormat.d.ts

@@ -3,12 +3,15 @@
 export declare const KEY_COLUMN = "key";
 export declare const EMPTY_BUFFER: Buffer;
 export declare const ABSENT: unique symbol;
-export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[]): Buffer[];
+export declare const TARGET_FILE_BYTES: number;
+export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes?: number): Buffer[];
 export type BaseBulkDatabaseReader = {
     rowCount: number;
     totalBytes: number;
     minTime: number;
     maxTime: number;
+    minKey?: string;
+    maxKey?: string;
     keys: string[];
     columns: {
         column: string;
@@ -26,6 +29,18 @@ export type BaseBulkDatabaseReader = {
         time: number;
     } | typeof ABSENT>;
 };
+export type BulkHeaderInfo = {
+    rowCount: number;
+    minTime: number;
+    maxTime: number;
+    minKey?: string;
+    maxKey?: string;
+    columns: {
+        column: string;
+        byteSize: number;
+    }[];
+};
+export declare function loadBulkHeader(getRange: (start: number, end: number) => Promise<Buffer>, totalBytes: number): Promise<BulkHeaderInfo>;
 export declare function loadBulkDatabase(config: {
     totalBytes: number;
     getRange: (start: number, end: number) => Promise<Buffer>;

package/storage/BulkDatabase2/BulkDatabaseFormat.ts

@@ -51,10 +51,15 @@ type FileHeader = {
     rowCount: number;
     columns: { name: string; offset: number; length: number }[];
     // Oldest/newest write-time of the data in this file (from the stream entries it was folded from,
-    // carried through merges). Lets the reader order bulk files by actual data recency and lets merges
-    // assert that two files' time ranges never overlap. Absent (0) in files written before this existed.
+    // carried through merges). Lets the merge planner pick files overlapping a time range. Absent (0)
+    // in files written before this existed.
     minTime?: number;
     maxTime?: number;
+    // Lexicographically smallest/largest key in this file (rows are stored key-sorted). Lets the merge
+    // planner group/dedup by key range and lets a single-key read skip files whose range excludes the
+    // key. Absent (undefined) in files written before this existed — treated as "spans all keys".
+    minKey?: string;
+    maxKey?: string;
 };
 
 function encodeValue(value: unknown): { type: number; bytes: Buffer } {
@@ -155,12 +160,12 @@ function decodeBulkData(blob: Buffer, rowCount: number): unknown[] {
     return values;
 }
 
-// Past this estimated logical size a single rows array is split across multiple files, so no one
-// file (and therefore no single column blob, and no single Buffer.concat) ever approaches the
-// ~2GB Buffer length limit. The cap is deliberately the same as the merge cap: 800MB is the most
-// we'll ever hold for one file. The estimate is rough on purpose — it only needs to keep each
-// chunk comfortably under the limit, not be exact.
-const FILE_SPLIT_BYTES = 800 * 1024 * 1024;
+// Target logical (uncompressed) size of one bulk file. A rows array bigger than this is split across
+// multiple key-contiguous files, so files stay around this size (the merge policy's target) and no
+// single column blob / Buffer.concat ever approaches the ~2GB Buffer length limit. The estimate is
+// rough on purpose — it only needs to keep each chunk near the target, not be exact. A single row
+// bigger than the target still becomes its own (oversized) file, since we never split within a key.
+export const TARGET_FILE_BYTES = 256 * 1024 * 1024;
 
 // Per-value on-disk overhead: a 4-byte offset entry plus a 1-byte type tag.
 const PER_VALUE_OVERHEAD = 5;
@@ -207,42 +212,61 @@ function buildOneFile(rows: Record<string, unknown>[], times: number[]): Buffer
     let minTime = times.length ? times[0] : 0;
     let maxTime = minTime;
     for (const t of times) { if (t < minTime) minTime = t; if (t > maxTime) maxTime = t; }
-    const header: FileHeader = { rowCount: rows.length, columns, minTime, maxTime };
+    let minKey: string | undefined;
+    let maxKey: string | undefined;
+    for (const row of rows) {
+        const key = row[KEY_COLUMN] as string;
+        if (minKey === undefined || key < minKey) minKey = key;
+        if (maxKey === undefined || key > maxKey) maxKey = key;
+    }
+    const header: FileHeader = { rowCount: rows.length, columns, minTime, maxTime, minKey, maxKey };
     const headerBuf = Buffer.from(JSON.stringify(header), "utf8");
     const lengthPrefix = Buffer.alloc(4);
     lengthPrefix.writeUInt32LE(headerBuf.length, 0);
     return Buffer.concat([lengthPrefix, headerBuf, ...blobs]);
 }
 
-// Returns one complete, independent file buffer per chunk of rows. When the caller hands us more
-// rows than fit comfortably in one file we partition by row range — each returned buffer is exactly
-// what buildOneFile would produce if called with that subset, so the chunks have disjoint keys and
-// the caller just writes each as its own file. A normal-sized write returns a single buffer.
+// Returns one complete, independent file buffer per chunk of rows. Rows are first sorted by key, then
+// partitioned into key-contiguous chunks of ~targetBytes each — so every returned file is key-sorted
+// (tight minKey/maxKey for the read-skip + merge planner) and stays near the target size, and no
+// single column blob / Buffer.concat approaches the ~2GB limit. The chunks have disjoint key ranges,
+// so the caller just writes each as its own file. A normal-sized write returns a single buffer.
 // `times[i]` is row i's write-time, stored per row so reads resolve a key to its latest value by time.
-export function buildFileBuffer(rows: Record<string, unknown>[], times: number[]): Buffer[] {
+export function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes = TARGET_FILE_BYTES): Buffer[] {
     if (rows.length === 0) return [buildOneFile([], [])];
+    // Sort rows + their times together by key so each output file is key-contiguous.
+    const order = rows.map((_, i) => i).sort((a, b) => {
+        const ka = rows[a][KEY_COLUMN] as string;
+        const kb = rows[b][KEY_COLUMN] as string;
+        return ka < kb ? -1 : ka > kb ? 1 : 0;
+    });
+    const sortedRows = order.map(i => rows[i]);
+    const sortedTimes = order.map(i => times[i]);
     const result: Buffer[] = [];
     let chunkStart = 0;
     let chunkBytes = 0;
-    for (let i = 0; i < rows.length; i++) {
-        const rowBytes = estimateRowBytes(rows[i]);
-        if (i > chunkStart && chunkBytes + rowBytes > FILE_SPLIT_BYTES) {
-            result.push(buildOneFile(rows.slice(chunkStart, i), times.slice(chunkStart, i)));
+    for (let i = 0; i < sortedRows.length; i++) {
+        const rowBytes = estimateRowBytes(sortedRows[i]);
+        if (i > chunkStart && chunkBytes + rowBytes > targetBytes) {
+            result.push(buildOneFile(sortedRows.slice(chunkStart, i), sortedTimes.slice(chunkStart, i)));
             chunkStart = i;
             chunkBytes = 0;
         }
         chunkBytes += rowBytes;
     }
-    result.push(buildOneFile(rows.slice(chunkStart), times.slice(chunkStart)));
+    result.push(buildOneFile(sortedRows.slice(chunkStart), sortedTimes.slice(chunkStart)));
     return result;
 }
 
 export type BaseBulkDatabaseReader = {
     rowCount: number;
     totalBytes: number;
-    // Write-time bounds of this reader's data (0 if unknown — old bulk files). Diagnostics only now.
+    // Write-time bounds of this reader's data (0 if unknown — old bulk files).
     minTime: number;
     maxTime: number;
+    // Key-range bounds (undefined for old files / the stream reader — treat as "spans all keys").
+    minKey?: string;
+    maxKey?: string;
     // Keys is special, it's always automatically decoded, even though it is stored as a normal column
     keys: string[];
     columns: { column: string; byteSize: number }[];
@@ -259,6 +283,25 @@ export type BaseBulkDatabaseReader = {
     getSingleField: (key: string, column: string) => Promise<{ value: unknown; time: number } | typeof ABSENT>;
 };
 
+// Reads just the file header (the 4-byte length + header JSON) — no column data. Used by the merge
+// planner to get each file's row count, time range, and key range cheaply across many files.
+export type BulkHeaderInfo = { rowCount: number; minTime: number; maxTime: number; minKey?: string; maxKey?: string; columns: { column: string; byteSize: number }[] };
+export async function loadBulkHeader(getRange: (start: number, end: number) => Promise<Buffer>, totalBytes: number): Promise<BulkHeaderInfo> {
+    const headerLength = (await getRange(0, 4)).readUInt32LE(0);
+    if (headerLength <= 0 || headerLength > totalBytes) {
+        throw new Error(`Expected header length in (0, ${totalBytes}], was ${headerLength}`);
+    }
+    const header = JSON.parse((await getRange(4, 4 + headerLength)).toString("utf8")) as FileHeader;
+    return {
+        rowCount: header.rowCount,
+        minTime: header.minTime || 0,
+        maxTime: header.maxTime || 0,
+        minKey: header.minKey,
+        maxKey: header.maxKey,
+        columns: header.columns.filter(c => c.name !== TIME_COLUMN).map(c => ({ column: c.name, byteSize: c.length })),
+    };
+}
+
 export async function loadBulkDatabase(config: {
     totalBytes: number;
     getRange: (start: number, end: number) => Promise<Buffer>;
@@ -300,6 +343,8 @@ export async function loadBulkDatabase(config: {
         totalBytes: config.totalBytes,
         minTime: header.minTime || 0,
         maxTime: header.maxTime || 0,
+        minKey: header.minKey,
+        maxKey: header.maxKey,
         keys,
         keyTimes: new Map(keys.map((key, i) => [key, times[i]])),
         columns: header.columns.filter(c => c.name !== TIME_COLUMN).map(c => ({ column: c.name, byteSize: c.length })),

package/storage/BulkDatabase2/mergeLock.ts

@@ -1,8 +1,9 @@
-// Best-effort "only one tab merges this collection at a time" guard, layered ON TOP of the manifest
-// scheme (which guarantees correctness on its own). This is purely an efficiency measure: it stops two
-// tabs doing the same compaction at once and racing to orphan each other's output. It uses
-// localStorage (shared across same-origin tabs) and is a no-op where localStorage is unavailable
-// (Node) — there the manifest backstop alone keeps things correct, which is also what the Node stress
+// Best-effort "only one tab merges this collection at a time" guard. This is purely an efficiency
+// measure: correctness does NOT depend on it. Reads resolve by per-row write-time, merges write new
+// files before deleting consumed ones, and a duplicate-heavy region gets re-merged — so two tabs
+// merging at once only waste work and briefly duplicate data, never corrupt or lose it. The lock just
+// stops that wasted work. It uses localStorage (shared across same-origin tabs) and is a no-op where
+// localStorage is unavailable (Node) — there concurrent merges are harmless, which the Node stress
 // tests exercise. localStorage has no atomic compare-and-swap, so we write-then-reread to shrink the
 // race window, and a TTL frees a lock left behind by a tab that crashed or closed mid-merge.
 

package/storage/BulkDatabase2/syncClient.d.ts

@@ -5,5 +5,6 @@ export type RemoteWrite = {
     value?: unknown;
 };
 export declare function isSyncSupported(): boolean;
-export declare function connect(collection: string, onWrite: (write: RemoteWrite) => void): Promise<RemoteWrite[]>;
+export declare function connect(collection: string, onWrite: (write: RemoteWrite) => void, onSeal?: () => void): Promise<RemoteWrite[]>;
 export declare function broadcast(collection: string, write: RemoteWrite): void;
+export declare function broadcastSeal(collection: string): void;

package/storage/BulkDatabase2/syncClient.ts

@@ -15,6 +15,8 @@ const RECENT_WINDOW_MS = 60_000;
 type Channel = {
     bc: BroadcastChannel;
     subscribers: ((write: RemoteWrite) => void)[];
+    // Called when a peer asks everyone to seal (stop appending to) their current stream file.
+    sealSubscribers: (() => void)[];
     // This tab's own recent writes, kept so it can answer another tab's "hello".
     recent: RemoteWrite[];
 };
@@ -41,7 +43,7 @@ function ensure(collection: string): Channel | undefined {
     // BroadcastChannel never delivers a message back to the instance that sent it, so a tab never
     // hears its own writes — only the other open tabs do.
     const bc = new BroadcastChannel(`bulkDatabase2:${collection}`);
-    const created: Channel = { bc, subscribers: [], recent: [] };
+    const created: Channel = { bc, subscribers: [], sealSubscribers: [], recent: [] };
     bc.onmessage = (event: MessageEvent) => {
         const msg = event.data as { type: string; write?: RemoteWrite; writes?: RemoteWrite[] };
         if (!msg) return;
@@ -49,6 +51,10 @@ function ensure(collection: string): Channel | undefined {
             deliver(created, msg.write);
         } else if (msg.type === "recent" && msg.writes) {
             for (const write of msg.writes) deliver(created, write);
+        } else if (msg.type === "seal") {
+            // A peer is about to fold recent data; stop appending to our current stream file so it
+            // becomes complete and the merge can fold it whole. Our next write starts a fresh file.
+            for (const sub of created.sealSubscribers) sub();
         } else if (msg.type === "hello") {
             // Another tab just started; replay our recent writes so it doesn't miss any. The reply goes
             // to every tab, but peers that already have a write ignore it (its timestamp isn't newer
@@ -64,10 +70,11 @@ function ensure(collection: string): Channel | undefined {
 // Subscribe to remote writes for a collection. Recent writes from already-open tabs arrive through the
 // same onWrite callback (as the reply to our hello), so the returned array is always empty — it's kept
 // only for API compatibility with callers that await it.
-export function connect(collection: string, onWrite: (write: RemoteWrite) => void): Promise<RemoteWrite[]> {
+export function connect(collection: string, onWrite: (write: RemoteWrite) => void, onSeal?: () => void): Promise<RemoteWrite[]> {
     const channel = ensure(collection);
     if (!channel) return Promise.resolve([]);
     channel.subscribers.push(onWrite);
+    if (onSeal) channel.sealSubscribers.push(onSeal);
     channel.bc.postMessage({ type: "hello" });
     return Promise.resolve([]);
 }
@@ -79,3 +86,13 @@ export function broadcast(collection: string, write: RemoteWrite): void {
     pruneRecent(channel);
     channel.bc.postMessage({ type: "write", write });
 }
+
+// Ask every other open tab of this collection to seal (stop appending to) its current stream file, so
+// a merge can fold recent data up to the present without racing an append. Best-effort: no-op in Node,
+// and a peer that misses it just keeps appending — worst case the merge folds a prefix and the rest
+// folds later (duplication, resolved by write-time), never data loss.
+export function broadcastSeal(collection: string): void {
+    const channel = ensure(collection);
+    if (!channel) return;
+    channel.bc.postMessage({ type: "seal" });
+}

package/storage/FileFolderAPI.tsx

@@ -183,7 +183,12 @@ export class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
             // Create the file
             await fs.promises.writeFile(filePath, Buffer.alloc(0));
         } else if (!exists) {
-            throw new Error(`File not found: ${filePath}`);
+            // Tag as ENOENT so readWithRetry treats it as a genuinely-missing file (return undefined now)
+            // rather than a transient read failure to retry 6× with backoff — missing files are normal
+            // (a concurrent merge deletes a file mid-read), and retrying them is catastrophically slow.
+            const err = new Error(`File not found: ${filePath}`) as Error & { code?: string };
+            err.code = "ENOENT";
+            throw err;
         }
 
         return new NodeJSFileHandleWrapper(filePath);

package/storage/BulkDatabase2/manifest.d.ts

@@ -1,17 +0,0 @@
-export declare const MANIFEST_EXTENSION = ".manifest";
-export type Manifest = {
-    startTime: number;
-    validBulkFiles: string[];
-    ignoredStreamFiles: string[];
-    readFiles: string[];
-};
-export declare function isManifestName(name: string): boolean;
-export declare function manifestFileName(startTime: number, writerId: string, counter: number): string;
-export declare function parseManifestStartTime(name: string): number | undefined;
-export declare function chooseManifest(manifests: {
-    name: string;
-    manifest: Manifest;
-}[]): {
-    name: string;
-    manifest: Manifest;
-} | undefined;

package/storage/BulkDatabase2/manifest.ts

@@ -1,59 +0,0 @@
-// A manifest records which bulk files are valid, decoupling "a file exists on disk" from "a file is
-// part of the database." Every operation that changes the bulk layout (rollover, merge, direct write)
-// writes a brand-new manifest instead of mutating/deleting in place, so changes are atomic from a
-// reader's point of view: a reader sees either the old manifest or the new one, never a half-applied
-// state. Manifests are immutable once written and never clobbered.
-//
-// Resolution: read every manifest, pick the one with the newest startTime (the time its writer
-// snapshotted the directory). The latest starter has the most up-to-date view, so its decision wins;
-// an older-starting writer that finishes later is ignored, and its freshly-written files are simply
-// orphaned (cleaned up later) — its inputs were never marked consumed, so no data is lost.
-//
-// Back-compat: if there is no manifest at all, every bulk file is valid (old databases just work).
-// Stream files are always valid unless a manifest lists them as already merged into a bulk file
-// (ignoredStreamFiles); they carry their own per-write timestamps, so they self-resolve regardless.
-
-export const MANIFEST_EXTENSION = ".manifest";
-
-export type Manifest = {
-    // When the writer snapshotted the directory (its read time). Newest startTime wins.
-    startTime: number;
-    // The full set of bulk files that are valid as of this manifest (not a delta).
-    validBulkFiles: string[];
-    // Stream files already folded into a bulk file — ignore them on read; cleanup deletes them later.
-    ignoredStreamFiles: string[];
-    // The filenames the writer saw at startTime (diagnostics + lets cleanup reason about what existed).
-    readFiles: string[];
-};
-
-export function isManifestName(name: string): boolean {
-    return name.endsWith(MANIFEST_EXTENSION);
-}
-
-// manifest_<startTime>_<writerId>_<counter>.manifest — writerId keeps two processes from colliding on
-// a name when they start in the same millisecond.
-export function manifestFileName(startTime: number, writerId: string, counter: number): string {
-    return `manifest_${startTime}_${writerId}_${counter}${MANIFEST_EXTENSION}`;
-}
-
-export function parseManifestStartTime(name: string): number | undefined {
-    if (!name.endsWith(MANIFEST_EXTENSION)) return undefined;
-    const parts = name.slice(0, -MANIFEST_EXTENSION.length).split("_");
-    if (parts[0] !== "manifest") return undefined;
-    const startTime = parseInt(parts[1], 10);
-    return Number.isFinite(startTime) ? startTime : undefined;
-}
-
-// Picks the authoritative manifest: newest startTime, ties broken by name for determinism. Returns
-// undefined if there are none (callers then treat every bulk file as valid).
-export function chooseManifest(manifests: { name: string; manifest: Manifest }[]): { name: string; manifest: Manifest } | undefined {
-    let chosen: { name: string; manifest: Manifest } | undefined;
-    for (const entry of manifests) {
-        if (!chosen
-            || entry.manifest.startTime > chosen.manifest.startTime
-            || (entry.manifest.startTime === chosen.manifest.startTime && entry.name > chosen.name)) {
-            chosen = entry;
-        }
-    }
-    return chosen;
-}