sliftutils 1.3.0 → 1.3.1

package/storage/BulkDatabase2/BulkDatabaseFormat.d.ts

@@ -2,21 +2,29 @@
 /// <reference types="node" />
 export declare const KEY_COLUMN = "key";
 export declare const EMPTY_BUFFER: Buffer;
-export declare function buildFileBuffer(rows: Record<string, unknown>[]): Buffer[];
+export declare const ABSENT: unique symbol;
+export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[]): Buffer[];
 export type BaseBulkDatabaseReader = {
     rowCount: number;
     totalBytes: number;
+    minTime: number;
+    maxTime: number;
     keys: string[];
     columns: {
         column: string;
         byteSize: number;
     }[];
-    deletedKeys?: Set<string>;
+    keyTimes: Map<string, number>;
+    deleteTimes?: Map<string, number>;
     getColumn: (column: string) => Promise<{
         key: string;
         value: unknown;
+        time: number;
     }[]>;
-    getSingleField: (key: string, column: string) => Promise<unknown | undefined>;
+    getSingleField: (key: string, column: string) => Promise<{
+        value: unknown;
+        time: number;
+    } | typeof ABSENT>;
 };
 export declare function loadBulkDatabase(config: {
     totalBytes: number;

package/storage/BulkDatabase2/BulkDatabaseFormat.ts

@@ -20,6 +20,10 @@ const TYPE_INT32_ARRAY = 10;
 const TYPE_UINT32_ARRAY = 11;
 const TYPE_FLOAT32_ARRAY = 12;
 const TYPE_FLOAT64_ARRAY = 13;
+// A cell whose row never set this column at all — as opposed to TYPE_UNDEFINED, an explicitly stored
+// undefined. On read, ABSENT falls through to older readers for that column; a stored undefined stops
+// the fall-through (it's a real value that clears the column).
+const TYPE_ABSENT = 14;
 
 const TYPED_ARRAY_TYPES: { type: number; ctor: { new(buffer: ArrayBuffer): ArrayBufferView; BYTES_PER_ELEMENT: number; name: string } }[] = [
     { type: TYPE_INT8_ARRAY, ctor: Int8Array },
@@ -35,12 +39,28 @@ const TYPED_ARRAY_TYPES: { type: number; ctor: { new(buffer: ArrayBuffer): Array
 
 export const EMPTY_BUFFER = Buffer.alloc(0) as Buffer;
 
+// Sentinel a reader returns for a cell whose row never set this column, so the join can fall through
+// to an older reader for that column. Distinct from a stored undefined, which is a real clearing value.
+export const ABSENT = Symbol("absent");
+
+// Hidden per-row column holding each row's write-time (so reads can resolve a key to its latest value
+// by actual time). NUL-prefixed so it can't collide with a user column; excluded from `columns`.
+const TIME_COLUMN = String.fromCharCode(0) + "t";
+
 type FileHeader = {
     rowCount: number;
     columns: { name: string; offset: number; length: number }[];
+    // Oldest/newest write-time of the data in this file (from the stream entries it was folded from,
+    // carried through merges). Lets the reader order bulk files by actual data recency and lets merges
+    // assert that two files' time ranges never overlap. Absent (0) in files written before this existed.
+    minTime?: number;
+    maxTime?: number;
 };
 
 function encodeValue(value: unknown): { type: number; bytes: Buffer } {
+    if (value === ABSENT) {
+        return { type: TYPE_ABSENT, bytes: EMPTY_BUFFER };
+    }
     if (value === undefined || value === null) {
         return { type: TYPE_UNDEFINED, bytes: EMPTY_BUFFER };
     }
@@ -90,6 +110,7 @@ function decodeValue(type: number, bytes: Buffer): unknown {
         return bytes[0] === 1;
     }
     if (type === TYPE_OBJECT) return JSON.parse(bytes.toString("utf8"));
+    if (type === TYPE_ABSENT) return ABSENT;
     const entry = TYPED_ARRAY_TYPES.find(t => t.type === type);
     if (!entry) {
         throw new Error(`Expected a valid type tag, was ${type}`);
@@ -162,7 +183,7 @@ function estimateRowBytes(row: Record<string, unknown>): number {
     return total;
 }
 
-function buildOneFile(rows: Record<string, unknown>[]): Buffer {
+function buildOneFile(rows: Record<string, unknown>[], times: number[]): Buffer {
     const columnNames: string[] = [];
     const columnSet = new Set<string>();
     for (const row of rows) {
@@ -172,14 +193,21 @@ function buildOneFile(rows: Record<string, unknown>[]): Buffer {
             columnNames.push(field);
         }
     }
-    const blobs = columnNames.map(col => encodeBulkData(rows.map(row => row[col])));
+    // A row that doesn't include a column stores ABSENT (fall-through), not undefined (a real value).
+    const blobs = columnNames.map(col => encodeBulkData(rows.map(row => col in row ? row[col] : ABSENT)));
+    // The hidden per-row time column.
+    columnNames.push(TIME_COLUMN);
+    blobs.push(encodeBulkData(times));
     let offset = 0;
     const columns = columnNames.map((name, i) => {
         const entry = { name, offset, length: blobs[i].length };
         offset += blobs[i].length;
         return entry;
     });
-    const header: FileHeader = { rowCount: rows.length, columns };
+    let minTime = times.length ? times[0] : 0;
+    let maxTime = minTime;
+    for (const t of times) { if (t < minTime) minTime = t; if (t > maxTime) maxTime = t; }
+    const header: FileHeader = { rowCount: rows.length, columns, minTime, maxTime };
     const headerBuf = Buffer.from(JSON.stringify(header), "utf8");
     const lengthPrefix = Buffer.alloc(4);
     lengthPrefix.writeUInt32LE(headerBuf.length, 0);
@@ -190,38 +218,45 @@ function buildOneFile(rows: Record<string, unknown>[]): Buffer {
 // rows than fit comfortably in one file we partition by row range — each returned buffer is exactly
 // what buildOneFile would produce if called with that subset, so the chunks have disjoint keys and
 // the caller just writes each as its own file. A normal-sized write returns a single buffer.
-export function buildFileBuffer(rows: Record<string, unknown>[]): Buffer[] {
-    if (rows.length === 0) return [buildOneFile([])];
+// `times[i]` is row i's write-time, stored per row so reads resolve a key to its latest value by time.
+export function buildFileBuffer(rows: Record<string, unknown>[], times: number[]): Buffer[] {
+    if (rows.length === 0) return [buildOneFile([], [])];
     const result: Buffer[] = [];
     let chunkStart = 0;
     let chunkBytes = 0;
     for (let i = 0; i < rows.length; i++) {
         const rowBytes = estimateRowBytes(rows[i]);
         if (i > chunkStart && chunkBytes + rowBytes > FILE_SPLIT_BYTES) {
-            result.push(buildOneFile(rows.slice(chunkStart, i)));
+            result.push(buildOneFile(rows.slice(chunkStart, i), times.slice(chunkStart, i)));
             chunkStart = i;
             chunkBytes = 0;
         }
         chunkBytes += rowBytes;
     }
-    result.push(buildOneFile(rows.slice(chunkStart)));
+    result.push(buildOneFile(rows.slice(chunkStart), times.slice(chunkStart)));
     return result;
 }
 
 export type BaseBulkDatabaseReader = {
     rowCount: number;
     totalBytes: number;
+    // Write-time bounds of this reader's data (0 if unknown — old bulk files). Diagnostics only now.
+    minTime: number;
+    maxTime: number;
     // Keys is special, it's always automatically decoded, even though it is stored as a normal column
     keys: string[];
     columns: { column: string; byteSize: number }[];
-    // Keys this reader tombstones (deleted). A newer reader's deletion suppresses the key in all
-    // older readers. Bulk readers never set this; the tier-0 stream reader does.
-    deletedKeys?: Set<string>;
-    getColumn: (column: string) => Promise<{
-        key: string;
-        value: unknown;
-    }[]>;
-    getSingleField: (key: string, column: string) => Promise<unknown | undefined>;
+    // Each key's row write-time (the time of its newest write in this reader). The join compares these
+    // across readers to resolve a key to its latest value.
+    keyTimes: Map<string, number>;
+    // Per-key tombstone time: the key was deleted at this time. The join treats a delete like any other
+    // event — a delete only wins if it's newer than every set for the key. Only the stream reader sets it.
+    deleteTimes?: Map<string, number>;
+    // Each key's value for the column plus the row's write-time. value may be ABSENT (the row never set
+    // this column — the join then falls through to an older reader for that key/column).
+    getColumn: (column: string) => Promise<{ key: string; value: unknown; time: number }[]>;
+    // The value + write-time for (key, column), or ABSENT if this reader has no such cell.
+    getSingleField: (key: string, column: string) => Promise<{ value: unknown; time: number } | typeof ABSENT>;
 };
 
 export async function loadBulkDatabase(config: {
@@ -254,20 +289,29 @@ export async function loadBulkDatabase(config: {
     });
     const keyIndex = new Map(keys.map((key, i) => [key, i]));
 
+    // Per-row write-times. Old files (written before this column existed) fall back to the file's header
+    // time (or 0) for every row — fine, since such files predate concurrent-time resolution.
+    const times: number[] = colByName.has(TIME_COLUMN)
+        ? (await readWholeColumn(TIME_COLUMN)).map(v => typeof v === "number" ? v : 0)
+        : keys.map(() => header.maxTime || 0);
+
     return {
         rowCount,
         totalBytes: config.totalBytes,
+        minTime: header.minTime || 0,
+        maxTime: header.maxTime || 0,
         keys,
-        columns: header.columns.map(c => ({ column: c.name, byteSize: c.length })),
+        keyTimes: new Map(keys.map((key, i) => [key, times[i]])),
+        columns: header.columns.filter(c => c.name !== TIME_COLUMN).map(c => ({ column: c.name, byteSize: c.length })),
         async getColumn(column) {
             const values = await readWholeColumn(column);
-            return keys.map((key, i) => ({ key, value: values[i] }));
+            return keys.map((key, i) => ({ key, value: values[i], time: times[i] }));
         },
         async getSingleField(key, column) {
             const row = keyIndex.get(key);
-            if (row === undefined) return undefined;
+            if (row === undefined) return ABSENT;
             const col = colByName.get(column);
-            if (!col) return undefined;
+            if (!col) return ABSENT;
             const colBase = dataBase + col.offset;
             const offsetsBuf = await config.getRange(colBase + 4 * row, colBase + 4 * row + 8);
             const start = offsetsBuf.readUInt32LE(0);
@@ -279,7 +323,9 @@ export async function loadBulkDatabase(config: {
             if (end > start) {
                 bytes = await config.getRange(dataStart + start, dataStart + end);
             }
-            return decodeValue(typeBuf[0], bytes);
+            const value = decodeValue(typeBuf[0], bytes);
+            if (value === ABSENT) return ABSENT;
+            return { value, time: times[row] };
         },
     };
 }

package/storage/BulkDatabase2/manifest.d.ts

@@ -0,0 +1,17 @@
+export declare const MANIFEST_EXTENSION = ".manifest";
+export type Manifest = {
+    startTime: number;
+    validBulkFiles: string[];
+    ignoredStreamFiles: string[];
+    readFiles: string[];
+};
+export declare function isManifestName(name: string): boolean;
+export declare function manifestFileName(startTime: number, writerId: string, counter: number): string;
+export declare function parseManifestStartTime(name: string): number | undefined;
+export declare function chooseManifest(manifests: {
+    name: string;
+    manifest: Manifest;
+}[]): {
+    name: string;
+    manifest: Manifest;
+} | undefined;

package/storage/BulkDatabase2/manifest.ts

@@ -0,0 +1,59 @@
+// A manifest records which bulk files are valid, decoupling "a file exists on disk" from "a file is
+// part of the database." Every operation that changes the bulk layout (rollover, merge, direct write)
+// writes a brand-new manifest instead of mutating/deleting in place, so changes are atomic from a
+// reader's point of view: a reader sees either the old manifest or the new one, never a half-applied
+// state. Manifests are immutable once written and never clobbered.
+//
+// Resolution: read every manifest, pick the one with the newest startTime (the time its writer
+// snapshotted the directory). The latest starter has the most up-to-date view, so its decision wins;
+// an older-starting writer that finishes later is ignored, and its freshly-written files are simply
+// orphaned (cleaned up later) — its inputs were never marked consumed, so no data is lost.
+//
+// Back-compat: if there is no manifest at all, every bulk file is valid (old databases just work).
+// Stream files are always valid unless a manifest lists them as already merged into a bulk file
+// (ignoredStreamFiles); they carry their own per-write timestamps, so they self-resolve regardless.
+
+export const MANIFEST_EXTENSION = ".manifest";
+
+export type Manifest = {
+    // When the writer snapshotted the directory (its read time). Newest startTime wins.
+    startTime: number;
+    // The full set of bulk files that are valid as of this manifest (not a delta).
+    validBulkFiles: string[];
+    // Stream files already folded into a bulk file — ignore them on read; cleanup deletes them later.
+    ignoredStreamFiles: string[];
+    // The filenames the writer saw at startTime (diagnostics + lets cleanup reason about what existed).
+    readFiles: string[];
+};
+
+export function isManifestName(name: string): boolean {
+    return name.endsWith(MANIFEST_EXTENSION);
+}
+
+// manifest_<startTime>_<writerId>_<counter>.manifest — writerId keeps two processes from colliding on
+// a name when they start in the same millisecond.
+export function manifestFileName(startTime: number, writerId: string, counter: number): string {
+    return `manifest_${startTime}_${writerId}_${counter}${MANIFEST_EXTENSION}`;
+}
+
+export function parseManifestStartTime(name: string): number | undefined {
+    if (!name.endsWith(MANIFEST_EXTENSION)) return undefined;
+    const parts = name.slice(0, -MANIFEST_EXTENSION.length).split("_");
+    if (parts[0] !== "manifest") return undefined;
+    const startTime = parseInt(parts[1], 10);
+    return Number.isFinite(startTime) ? startTime : undefined;
+}
+
+// Picks the authoritative manifest: newest startTime, ties broken by name for determinism. Returns
+// undefined if there are none (callers then treat every bulk file as valid).
+export function chooseManifest(manifests: { name: string; manifest: Manifest }[]): { name: string; manifest: Manifest } | undefined {
+    let chosen: { name: string; manifest: Manifest } | undefined;
+    for (const entry of manifests) {
+        if (!chosen
+            || entry.manifest.startTime > chosen.manifest.startTime
+            || (entry.manifest.startTime === chosen.manifest.startTime && entry.name > chosen.name)) {
+            chosen = entry;
+        }
+    }
+    return chosen;
+}

package/storage/BulkDatabase2/mergeLock.d.ts

@@ -0,0 +1,2 @@
+export declare function tryAcquireMergeLock(collection: string, holderId: string): boolean;
+export declare function releaseMergeLock(collection: string, holderId: string): void;

package/storage/BulkDatabase2/mergeLock.ts

@@ -0,0 +1,47 @@
+// Best-effort "only one tab merges this collection at a time" guard, layered ON TOP of the manifest
+// scheme (which guarantees correctness on its own). This is purely an efficiency measure: it stops two
+// tabs doing the same compaction at once and racing to orphan each other's output. It uses
+// localStorage (shared across same-origin tabs) and is a no-op where localStorage is unavailable
+// (Node) — there the manifest backstop alone keeps things correct, which is also what the Node stress
+// tests exercise. localStorage has no atomic compare-and-swap, so we write-then-reread to shrink the
+// race window, and a TTL frees a lock left behind by a tab that crashed or closed mid-merge.
+
+const LOCK_TTL_MS = 30 * 1000;
+
+function getLocalStorage(): Storage | undefined {
+    try {
+        return typeof localStorage !== "undefined" ? localStorage : undefined;
+    } catch {
+        return undefined; // accessing localStorage can throw (e.g. disabled cookies)
+    }
+}
+
+function lockKey(collection: string): string {
+    return "bulkDatabase2-merge:" + collection;
+}
+
+// Returns true if we now hold the lock (or there's no localStorage to coordinate through, in which
+// case we proceed and rely on the manifest backstop). Returns false if another tab holds a fresh lock.
+export function tryAcquireMergeLock(collection: string, holderId: string): boolean {
+    const ls = getLocalStorage();
+    if (!ls) return true;
+    const key = lockKey(collection);
+    const now = Date.now();
+    const existing = ls.getItem(key);
+    if (existing) {
+        const t = parseInt(existing.slice(existing.lastIndexOf(":") + 1), 10);
+        if (Number.isFinite(t) && now - t < LOCK_TTL_MS) return false;
+    }
+    const token = holderId + ":" + now;
+    ls.setItem(key, token);
+    // Re-read to catch a racing setItem from another tab (best-effort, not atomic).
+    return ls.getItem(key) === token;
+}
+
+export function releaseMergeLock(collection: string, holderId: string): void {
+    const ls = getLocalStorage();
+    if (!ls) return;
+    const key = lockKey(collection);
+    const existing = ls.getItem(key);
+    if (existing && existing.startsWith(holderId + ":")) ls.removeItem(key);
+}

package/storage/BulkDatabase2/streamLog.ts

@@ -1,5 +1,5 @@
 import cborx from "cbor-x";
-import { BaseBulkDatabaseReader } from "./BulkDatabaseFormat";
+import { ABSENT, BaseBulkDatabaseReader } from "./BulkDatabaseFormat";
 
 // Tier-0 streaming format: an append log of whole-row writes and deletes (row-format, not columnar),
 // so small mutations are a single cheap append instead of rewriting a columnar file. Each block is:
@@ -64,10 +64,12 @@ export function parseStream(buffer: Buffer): { entries: StreamEntry[]; badBytes:
     return { entries, badBytes: buffer.length - pos };
 }
 
-// Wraps streamed entries (already ordered oldest-first) as a BaseBulkDatabaseReader. Applies
-// set/delete newest-wins: later entries overwrite earlier ones, and a delete tombstones the key
-// (exposed via deletedKeys so the join suppresses it in older bulk readers). Also returns the latest
-// timestamp seen per key (live or deleted), used for cross-tab conflict resolution.
+// Wraps streamed entries (already ordered oldest-first) as a BaseBulkDatabaseReader. Each set MERGES
+// its fields onto the key's current row (so a partial write/update only changes the columns it
+// includes); a delete tombstones the key (exposed via deletedKeys so the join suppresses it in older
+// bulk readers) and resets the merge. A column the merged row never set reads as ABSENT, so the join
+// falls through to older readers for it. Also returns the latest timestamp seen per key (live or
+// deleted), used for cross-tab conflict resolution.
 export function streamReaderFromEntries(entries: StreamEntry[], totalBytes: number): { reader: BaseBulkDatabaseReader; times: Map<string, number> } {
     let byKey = new Map<string, Record<string, unknown>>();
     let deletedKeys = new Set<string>();
@@ -79,7 +81,7 @@ export function streamReaderFromEntries(entries: StreamEntry[], totalBytes: numb
             times.set(entry.deletedKey, entry.time);
         } else if (entry.row) {
             let key = entry.row.key as string;
-            byKey.set(key, entry.row);
+            byKey.set(key, { ...byKey.get(key), ...entry.row });
             deletedKeys.delete(key);
             times.set(key, entry.time);
         }
@@ -95,21 +97,31 @@ export function streamReaderFromEntries(entries: StreamEntry[], totalBytes: numb
         }
     }
     let columns = columnNames.map(column => ({ column, byteSize: 0 }));
+    // Per-key tombstone times for the join's delete resolution.
+    let deleteTimes = new Map<string, number>();
+    for (let key of deletedKeys) deleteTimes.set(key, times.get(key) || 0);
+    let timeValues = [...times.values()];
     let reader: BaseBulkDatabaseReader = {
         totalBytes,
         rowCount: keys.length,
+        minTime: timeValues.length ? Math.min(...timeValues) : 0,
+        maxTime: timeValues.length ? Math.max(...timeValues) : 0,
         keys,
+        keyTimes: new Map(keys.map(key => [key, times.get(key) || 0])),
         columns,
-        deletedKeys,
+        deleteTimes,
+        // A key's time is its latest write across all columns (per-key, not per-column). For a column
+        // it never set we return ABSENT so the join falls through to an older reader.
         async getColumn(column) {
             return keys.map(key => {
                 let row = byKey.get(key);
-                return { key, value: row && row[column] };
+                return { key, value: row && column in row ? row[column] : ABSENT, time: times.get(key) || 0 };
             });
         },
         async getSingleField(key, column) {
             let row = byKey.get(key);
-            return row && row[column];
+            if (!row || !(column in row)) return ABSENT;
+            return { value: row[column], time: times.get(key) || 0 };
         },
     };
     return { reader, times };