sliftutils 1.3.0 → 1.4.0

package/index.d.ts

@@ -764,11 +764,93 @@ declare module "sliftutils/render-utils/observer" {
 
 declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
     import { BulkDatabaseBase } from "./BulkDatabaseBase";
-    export { BulkDatabaseBase, noopReactiveDeps } from "./BulkDatabaseBase";
+    export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
     export type { ReactiveDeps, StorageFactory } from "./BulkDatabaseBase";
+    /** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
+    export type BulkColumnInfo = {
+        column: string;
+        byteSize: number;
+    };
+    /** A snapshot of the collection's shape (no row data), as reported by getReaderInfo. */
+    export type BulkReaderInfo = {
+        rowCount: number;
+        totalBytes: number;
+        keyCount: number;
+        sampleKey: string | undefined;
+        columns: BulkColumnInfo[];
+    };
+    /**
+     * The full public API of BulkDatabase2 (the static `clearCache()` aside). BulkDatabase2 implements
+     * this; an application that just wants to depend on the surface can type against this interface instead
+     * of reading the implementation. `T` is the row type and must have a string `key`.
+     *
+     * Reads resolve every key/column by the latest write-time across all storage tiers. Writes are
+     * column-merges: a write/update only changes the columns it includes; columns it omits keep their
+     * previous value (clear a column by writing it as `undefined`).
+     */
+    export interface IBulkDatabase2<T extends {
+        key: string;
+    }> {
+        /** The collection name (its folder under the storage root). */
+        readonly name: string;
+        /** Write one full row (merging its columns onto any existing row for the key). */
+        write(entry: T): Promise<void>;
+        /** Write many full rows in one batch. */
+        writeBatch(entries: T[]): Promise<void>;
+        /** Update only the given columns of an existing key (key required). Warns and no-ops if the key isn't present. */
+        update(entry: Partial<T> & {
+            key: string;
+        }): Promise<void>;
+        /** Update many keys' partial columns in one batch. */
+        updateBatch(entries: (Partial<T> & {
+            key: string;
+        })[]): Promise<void>;
+        /** Delete a key. */
+        delete(key: string): Promise<void>;
+        /** Delete many keys in one batch. */
+        deleteBatch(keys: string[]): Promise<void>;
+        /** All live keys. */
+        getKeys(): Promise<string[]>;
+        /** One field's value for a key, or undefined if the key/column isn't set or the key is deleted. */
+        getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
+        /** A whole column as {key, value} for every live key. */
+        getColumn<Column extends keyof T>(column: Column): Promise<{
+            key: string;
+            value: T[Column];
+        }[]>;
+        /**
+         * Synchronous, reactive read of one field. Returns undefined while the base value is still loading
+         * (and re-renders once it arrives, under a mobx observer); reflects pending writes immediately.
+         */
+        getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
+        /** Synchronous, reactive read of a whole column. undefined while still loading. */
+        getColumnSync<Column extends keyof T>(column: Column): {
+            key: string;
+            value: T[Column];
+        }[] | undefined;
+        /** The columns present on disk and their byte sizes (no row data read). */
+        getColumnInfo(): Promise<BulkColumnInfo[]>;
+        /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
+        getReaderInfo(): Promise<BulkReaderInfo>;
+        /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
+        compact(): Promise<void>;
+        /**
+         * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
+         * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and
+         * whether it bailed because another tab/process holds the merge lock — so a scheduler can call this
+         * (e.g. every 30 minutes) and tell "nothing to do" from "someone else is already merging".
+         */
+        tryMergeNow(): Promise<{
+            merged: boolean;
+            lockFailed: boolean;
+        }>;
+        /** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
+         * most callers want compact() or tryMergeNow(). */
+        merge(timeLo: number, timeHi: number): Promise<void>;
+    }
     export declare class BulkDatabase2<T extends {
         key: string;
-    }> extends BulkDatabaseBase<T> {
+    }> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {
         constructor(name: string);
     }
 
@@ -776,6 +858,12 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
 
 declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
     import type { FileStorage } from "../FileFolderAPI";
+    export declare const bulkDatabase2Timing: {
+        streamSealAgeMs: number;
+        mergeCheckIntervalMs: number;
+        firstMergeTriggerFiles: number;
+        firstMergeTriggerRangeMs: number;
+    };
     export interface ReactiveDeps {
         observe(signal: string): void;
         invalidate(signal: string): void;
@@ -799,10 +887,13 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private overlay;
         private streamTimes;
         private streamFileName;
-        private streamRowsWritten;
+        private lastMergeCheck;
         private getStreamFileName;
-        private setOverlay;
+        private invalidateOverlay;
+        private setOverlayRow;
+        private setOverlayDeleted;
         private reader;
+        private buildReader;
         private syncSetup;
         private localTime;
         private applyRemote;
@@ -811,20 +902,32 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         writeBatch(entries: T[]): Promise<void>;
         delete(key: string): Promise<void>;
         deleteBatch(keys: string[]): Promise<void>;
+        update(entry: Partial<T> & {
+            key: string;
+        }): Promise<void>;
+        updateBatch(entries: (Partial<T> & {
+            key: string;
+        })[]): Promise<void>;
+        private listFiles;
         private writeBulkFile;
-        private listStreamFiles;
         private loadStreamEntries;
         private orderStreamEntries;
-        private maybeRolloverStream;
-        private rolloverStream;
+        private maybeMerge;
+        tryMergeNow(): Promise<{
+            merged: boolean;
+            lockFailed: boolean;
+        }>;
         compact(): Promise<void>;
-        private listFiles;
+        merge(timeLo: number, timeHi: number): Promise<void>;
         private makeRawGetRange;
         private loadFileReader;
+        private readBulkHeader;
         private fileLogicalSize;
         private handleUnreadableFile;
-        private mergeFilesBase;
-        private mergeFiles;
+        private resolveReaders;
+        private mergeFileSet;
+        private canDeleteStream;
+        private testMerge;
         private formatInfo;
         private patchColumn;
         getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
@@ -867,22 +970,45 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseFormat" {
     /// <reference types="node" />
     export declare const KEY_COLUMN = "key";
     export declare const EMPTY_BUFFER: Buffer;
-    export declare function buildFileBuffer(rows: Record<string, unknown>[]): Buffer[];
+    export declare const ABSENT: unique symbol;
+    export declare const TARGET_FILE_BYTES: number;
+    export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes?: number): Buffer[];
     export type BaseBulkDatabaseReader = {
         rowCount: number;
         totalBytes: number;
+        minTime: number;
+        maxTime: number;
+        minKey?: string;
+        maxKey?: string;
         keys: string[];
         columns: {
             column: string;
             byteSize: number;
         }[];
-        deletedKeys?: Set<string>;
+        keyTimes: Map<string, number>;
+        deleteTimes?: Map<string, number>;
         getColumn: (column: string) => Promise<{
             key: string;
             value: unknown;
+            time: number;
         }[]>;
-        getSingleField: (key: string, column: string) => Promise<unknown | undefined>;
+        getSingleField: (key: string, column: string) => Promise<{
+            value: unknown;
+            time: number;
+        } | typeof ABSENT>;
     };
+    export type BulkHeaderInfo = {
+        rowCount: number;
+        minTime: number;
+        maxTime: number;
+        minKey?: string;
+        maxKey?: string;
+        columns: {
+            column: string;
+            byteSize: number;
+        }[];
+    };
+    export declare function loadBulkHeader(getRange: (start: number, end: number) => Promise<Buffer>, totalBytes: number): Promise<BulkHeaderInfo>;
     export declare function loadBulkDatabase(config: {
         totalBytes: number;
         getRange: (start: number, end: number) => Promise<Buffer>;
@@ -911,6 +1037,12 @@ declare module "sliftutils/storage/BulkDatabase2/blockCache" {
 
 }
 
+declare module "sliftutils/storage/BulkDatabase2/mergeLock" {
+    export declare function tryAcquireMergeLock(collection: string, holderId: string): boolean;
+    export declare function releaseMergeLock(collection: string, holderId: string): void;
+
+}
+
 declare module "sliftutils/storage/BulkDatabase2/streamLog" {
     /// <reference types="node" />
     /// <reference types="node" />
@@ -948,8 +1080,9 @@ declare module "sliftutils/storage/BulkDatabase2/syncClient" {
         value?: unknown;
     };
     export declare function isSyncSupported(): boolean;
-    export declare function connect(collection: string, onWrite: (write: RemoteWrite) => void): Promise<RemoteWrite[]>;
+    export declare function connect(collection: string, onWrite: (write: RemoteWrite) => void, onSeal?: () => void): Promise<RemoteWrite[]>;
     export declare function broadcast(collection: string, write: RemoteWrite): void;
+    export declare function broadcastSeal(collection: string): void;
 
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.3.0",
+    "version": "1.4.0",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabase2.d.ts

@@ -1,8 +1,90 @@
 import { BulkDatabaseBase } from "./BulkDatabaseBase";
-export { BulkDatabaseBase, noopReactiveDeps } from "./BulkDatabaseBase";
+export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
 export type { ReactiveDeps, StorageFactory } from "./BulkDatabaseBase";
+/** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
+export type BulkColumnInfo = {
+    column: string;
+    byteSize: number;
+};
+/** A snapshot of the collection's shape (no row data), as reported by getReaderInfo. */
+export type BulkReaderInfo = {
+    rowCount: number;
+    totalBytes: number;
+    keyCount: number;
+    sampleKey: string | undefined;
+    columns: BulkColumnInfo[];
+};
+/**
+ * The full public API of BulkDatabase2 (the static `clearCache()` aside). BulkDatabase2 implements
+ * this; an application that just wants to depend on the surface can type against this interface instead
+ * of reading the implementation. `T` is the row type and must have a string `key`.
+ *
+ * Reads resolve every key/column by the latest write-time across all storage tiers. Writes are
+ * column-merges: a write/update only changes the columns it includes; columns it omits keep their
+ * previous value (clear a column by writing it as `undefined`).
+ */
+export interface IBulkDatabase2<T extends {
+    key: string;
+}> {
+    /** The collection name (its folder under the storage root). */
+    readonly name: string;
+    /** Write one full row (merging its columns onto any existing row for the key). */
+    write(entry: T): Promise<void>;
+    /** Write many full rows in one batch. */
+    writeBatch(entries: T[]): Promise<void>;
+    /** Update only the given columns of an existing key (key required). Warns and no-ops if the key isn't present. */
+    update(entry: Partial<T> & {
+        key: string;
+    }): Promise<void>;
+    /** Update many keys' partial columns in one batch. */
+    updateBatch(entries: (Partial<T> & {
+        key: string;
+    })[]): Promise<void>;
+    /** Delete a key. */
+    delete(key: string): Promise<void>;
+    /** Delete many keys in one batch. */
+    deleteBatch(keys: string[]): Promise<void>;
+    /** All live keys. */
+    getKeys(): Promise<string[]>;
+    /** One field's value for a key, or undefined if the key/column isn't set or the key is deleted. */
+    getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
+    /** A whole column as {key, value} for every live key. */
+    getColumn<Column extends keyof T>(column: Column): Promise<{
+        key: string;
+        value: T[Column];
+    }[]>;
+    /**
+     * Synchronous, reactive read of one field. Returns undefined while the base value is still loading
+     * (and re-renders once it arrives, under a mobx observer); reflects pending writes immediately.
+     */
+    getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
+    /** Synchronous, reactive read of a whole column. undefined while still loading. */
+    getColumnSync<Column extends keyof T>(column: Column): {
+        key: string;
+        value: T[Column];
+    }[] | undefined;
+    /** The columns present on disk and their byte sizes (no row data read). */
+    getColumnInfo(): Promise<BulkColumnInfo[]>;
+    /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
+    getReaderInfo(): Promise<BulkReaderInfo>;
+    /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
+    compact(): Promise<void>;
+    /**
+     * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
+     * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and
+     * whether it bailed because another tab/process holds the merge lock — so a scheduler can call this
+     * (e.g. every 30 minutes) and tell "nothing to do" from "someone else is already merging".
+     */
+    tryMergeNow(): Promise<{
+        merged: boolean;
+        lockFailed: boolean;
+    }>;
+    /** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
+     * most callers want compact() or tryMergeNow(). */
+    merge(timeLo: number, timeHi: number): Promise<void>;
+}
 export declare class BulkDatabase2<T extends {
     key: string;
-}> extends BulkDatabaseBase<T> {
+}> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {
     constructor(name: string);
 }

package/storage/BulkDatabase2/BulkDatabase2.ts

@@ -2,9 +2,83 @@ import { getFileStorageNested2 } from "../FileFolderAPI";
 import { observable, runInAction } from "../../render-utils/mobxTyped";
 import { BulkDatabaseBase, ReactiveDeps } from "./BulkDatabaseBase";
 
-export { BulkDatabaseBase, noopReactiveDeps } from "./BulkDatabaseBase";
+export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
 export type { ReactiveDeps, StorageFactory } from "./BulkDatabaseBase";
 
+/** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
+export type BulkColumnInfo = { column: string; byteSize: number };
+
+/** A snapshot of the collection's shape (no row data), as reported by getReaderInfo. */
+export type BulkReaderInfo = {
+    rowCount: number;
+    totalBytes: number;
+    keyCount: number;
+    sampleKey: string | undefined;
+    columns: BulkColumnInfo[];
+};
+
+/**
+ * The full public API of BulkDatabase2 (the static `clearCache()` aside). BulkDatabase2 implements
+ * this; an application that just wants to depend on the surface can type against this interface instead
+ * of reading the implementation. `T` is the row type and must have a string `key`.
+ *
+ * Reads resolve every key/column by the latest write-time across all storage tiers. Writes are
+ * column-merges: a write/update only changes the columns it includes; columns it omits keep their
+ * previous value (clear a column by writing it as `undefined`).
+ */
+export interface IBulkDatabase2<T extends { key: string }> {
+    /** The collection name (its folder under the storage root). */
+    readonly name: string;
+
+    /** Write one full row (merging its columns onto any existing row for the key). */
+    write(entry: T): Promise<void>;
+    /** Write many full rows in one batch. */
+    writeBatch(entries: T[]): Promise<void>;
+    /** Update only the given columns of an existing key (key required). Warns and no-ops if the key isn't present. */
+    update(entry: Partial<T> & { key: string }): Promise<void>;
+    /** Update many keys' partial columns in one batch. */
+    updateBatch(entries: (Partial<T> & { key: string })[]): Promise<void>;
+    /** Delete a key. */
+    delete(key: string): Promise<void>;
+    /** Delete many keys in one batch. */
+    deleteBatch(keys: string[]): Promise<void>;
+
+    /** All live keys. */
+    getKeys(): Promise<string[]>;
+    /** One field's value for a key, or undefined if the key/column isn't set or the key is deleted. */
+    getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
+    /** A whole column as {key, value} for every live key. */
+    getColumn<Column extends keyof T>(column: Column): Promise<{ key: string; value: T[Column] }[]>;
+
+    /**
+     * Synchronous, reactive read of one field. Returns undefined while the base value is still loading
+     * (and re-renders once it arrives, under a mobx observer); reflects pending writes immediately.
+     */
+    getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
+    /** Synchronous, reactive read of a whole column. undefined while still loading. */
+    getColumnSync<Column extends keyof T>(column: Column): { key: string; value: T[Column] }[] | undefined;
+
+    /** The columns present on disk and their byte sizes (no row data read). */
+    getColumnInfo(): Promise<BulkColumnInfo[]>;
+    /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
+    getReaderInfo(): Promise<BulkReaderInfo>;
+
+    /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
+    compact(): Promise<void>;
+
+    /**
+     * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
+     * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and
+     * whether it bailed because another tab/process holds the merge lock — so a scheduler can call this
+     * (e.g. every 30 minutes) and tell "nothing to do" from "someone else is already merging".
+     */
+    tryMergeNow(): Promise<{ merged: boolean; lockFailed: boolean }>;
+
+    /** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
+     * most callers want compact() or tryMergeNow(). */
+    merge(timeLo: number, timeHi: number): Promise<void>;
+}
+
 // mobx-backed reactivity: each signal string gets its own observable box. observe() reads it (so an
 // observer/autorun that calls it tracks that box) and invalidate() bumps it (so those reactions re-run).
 // This reproduces the fine-grained per-key + load-version reactivity the class had when it used
@@ -34,7 +108,7 @@ class MobxReactiveDeps implements ReactiveDeps {
 // Backwards-compatible BulkDatabase2: the mobx-reactive flavor. All behavior lives in BulkDatabaseBase;
 // this just supplies mobx reactivity and the default (getFileStorageNested2) storage backend, so the
 // sync reads (getSingleFieldSync / getColumnSync) stay observable for mobx components.
-export class BulkDatabase2<T extends { key: string }> extends BulkDatabaseBase<T> {
+export class BulkDatabase2<T extends { key: string }> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {
     constructor(name: string) {
         super(name, new MobxReactiveDeps(), getFileStorageNested2);
     }

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -1,4 +1,10 @@
 import type { FileStorage } from "../FileFolderAPI";
+export declare const bulkDatabase2Timing: {
+    streamSealAgeMs: number;
+    mergeCheckIntervalMs: number;
+    firstMergeTriggerFiles: number;
+    firstMergeTriggerRangeMs: number;
+};
 export interface ReactiveDeps {
     observe(signal: string): void;
     invalidate(signal: string): void;
@@ -22,10 +28,13 @@ export declare class BulkDatabaseBase<T extends {
     private overlay;
     private streamTimes;
     private streamFileName;
-    private streamRowsWritten;
+    private lastMergeCheck;
     private getStreamFileName;
-    private setOverlay;
+    private invalidateOverlay;
+    private setOverlayRow;
+    private setOverlayDeleted;
     private reader;
+    private buildReader;
     private syncSetup;
     private localTime;
     private applyRemote;
@@ -34,20 +43,32 @@ export declare class BulkDatabaseBase<T extends {
     writeBatch(entries: T[]): Promise<void>;
     delete(key: string): Promise<void>;
     deleteBatch(keys: string[]): Promise<void>;
+    update(entry: Partial<T> & {
+        key: string;
+    }): Promise<void>;
+    updateBatch(entries: (Partial<T> & {
+        key: string;
+    })[]): Promise<void>;
+    private listFiles;
     private writeBulkFile;
-    private listStreamFiles;
     private loadStreamEntries;
     private orderStreamEntries;
-    private maybeRolloverStream;
-    private rolloverStream;
+    private maybeMerge;
+    tryMergeNow(): Promise<{
+        merged: boolean;
+        lockFailed: boolean;
+    }>;
     compact(): Promise<void>;
-    private listFiles;
+    merge(timeLo: number, timeHi: number): Promise<void>;
     private makeRawGetRange;
     private loadFileReader;
+    private readBulkHeader;
     private fileLogicalSize;
     private handleUnreadableFile;
-    private mergeFilesBase;
-    private mergeFiles;
+    private resolveReaders;
+    private mergeFileSet;
+    private canDeleteStream;
+    private testMerge;
     private formatInfo;
     private patchColumn;
     getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;