sliftutils 1.3.0 → 1.3.1

package/index.d.ts

@@ -764,11 +764,80 @@ declare module "sliftutils/render-utils/observer" {
 
 declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
     import { BulkDatabaseBase } from "./BulkDatabaseBase";
-    export { BulkDatabaseBase, noopReactiveDeps } from "./BulkDatabaseBase";
+    export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
     export type { ReactiveDeps, StorageFactory } from "./BulkDatabaseBase";
+    /** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
+    export type BulkColumnInfo = {
+        column: string;
+        byteSize: number;
+    };
+    /** A snapshot of the collection's shape (no row data), as reported by getReaderInfo. */
+    export type BulkReaderInfo = {
+        rowCount: number;
+        totalBytes: number;
+        keyCount: number;
+        sampleKey: string | undefined;
+        columns: BulkColumnInfo[];
+    };
+    /**
+     * The full public API of BulkDatabase2 (the static `clearCache()` aside). BulkDatabase2 implements
+     * this; an application that just wants to depend on the surface can type against this interface instead
+     * of reading the implementation. `T` is the row type and must have a string `key`.
+     *
+     * Reads resolve every key/column by the latest write-time across all storage tiers. Writes are
+     * column-merges: a write/update only changes the columns it includes; columns it omits keep their
+     * previous value (clear a column by writing it as `undefined`).
+     */
+    export interface IBulkDatabase2<T extends {
+        key: string;
+    }> {
+        /** The collection name (its folder under the storage root). */
+        readonly name: string;
+        /** Write one full row (merging its columns onto any existing row for the key). */
+        write(entry: T): Promise<void>;
+        /** Write many full rows in one batch. */
+        writeBatch(entries: T[]): Promise<void>;
+        /** Update only the given columns of an existing key (key required). Warns and no-ops if the key isn't present. */
+        update(entry: Partial<T> & {
+            key: string;
+        }): Promise<void>;
+        /** Update many keys' partial columns in one batch. */
+        updateBatch(entries: (Partial<T> & {
+            key: string;
+        })[]): Promise<void>;
+        /** Delete a key. */
+        delete(key: string): Promise<void>;
+        /** Delete many keys in one batch. */
+        deleteBatch(keys: string[]): Promise<void>;
+        /** All live keys. */
+        getKeys(): Promise<string[]>;
+        /** One field's value for a key, or undefined if the key/column isn't set or the key is deleted. */
+        getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
+        /** A whole column as {key, value} for every live key. */
+        getColumn<Column extends keyof T>(column: Column): Promise<{
+            key: string;
+            value: T[Column];
+        }[]>;
+        /**
+         * Synchronous, reactive read of one field. Returns undefined while the base value is still loading
+         * (and re-renders once it arrives, under a mobx observer); reflects pending writes immediately.
+         */
+        getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
+        /** Synchronous, reactive read of a whole column. undefined while still loading. */
+        getColumnSync<Column extends keyof T>(column: Column): {
+            key: string;
+            value: T[Column];
+        }[] | undefined;
+        /** The columns present on disk and their byte sizes (no row data read). */
+        getColumnInfo(): Promise<BulkColumnInfo[]>;
+        /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
+        getReaderInfo(): Promise<BulkReaderInfo>;
+        /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
+        compact(): Promise<void>;
+    }
     export declare class BulkDatabase2<T extends {
         key: string;
-    }> extends BulkDatabaseBase<T> {
+    }> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {
         constructor(name: string);
     }
 
@@ -776,6 +845,14 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
 
 declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
     import type { FileStorage } from "../FileFolderAPI";
+    export declare const bulkDatabase2Timing: {
+        streamSealAgeMs: number;
+        foldDataAgeMs: number;
+        foldTriggerAgeMs: number;
+        foldCheckIntervalMs: number;
+        cleanupAgeMs: number;
+        cleanupIntervalMs: number;
+    };
     export interface ReactiveDeps {
         observe(signal: string): void;
         invalidate(signal: string): void;
@@ -799,9 +876,12 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private overlay;
         private streamTimes;
         private streamFileName;
-        private streamRowsWritten;
+        private lastCleanup;
+        private lastFoldCheck;
         private getStreamFileName;
-        private setOverlay;
+        private invalidateOverlay;
+        private setOverlayRow;
+        private setOverlayDeleted;
         private reader;
         private syncSetup;
         private localTime;
@@ -811,20 +891,28 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         writeBatch(entries: T[]): Promise<void>;
         delete(key: string): Promise<void>;
         deleteBatch(keys: string[]): Promise<void>;
+        update(entry: Partial<T> & {
+            key: string;
+        }): Promise<void>;
+        updateBatch(entries: (Partial<T> & {
+            key: string;
+        })[]): Promise<void>;
+        private getValidFiles;
+        private commitManifest;
         private writeBulkFile;
-        private listStreamFiles;
+        private consolidate;
         private loadStreamEntries;
         private orderStreamEntries;
         private maybeRolloverStream;
-        private rolloverStream;
         compact(): Promise<void>;
-        private listFiles;
         private makeRawGetRange;
         private loadFileReader;
         private fileLogicalSize;
         private handleUnreadableFile;
         private mergeFilesBase;
         private mergeFiles;
+        private mergeFilesLocked;
+        private cleanup;
         private formatInfo;
         private patchColumn;
         getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
@@ -867,21 +955,29 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseFormat" {
     /// <reference types="node" />
     export declare const KEY_COLUMN = "key";
     export declare const EMPTY_BUFFER: Buffer;
-    export declare function buildFileBuffer(rows: Record<string, unknown>[]): Buffer[];
+    export declare const ABSENT: unique symbol;
+    export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[]): Buffer[];
     export type BaseBulkDatabaseReader = {
         rowCount: number;
         totalBytes: number;
+        minTime: number;
+        maxTime: number;
         keys: string[];
         columns: {
             column: string;
             byteSize: number;
         }[];
-        deletedKeys?: Set<string>;
+        keyTimes: Map<string, number>;
+        deleteTimes?: Map<string, number>;
         getColumn: (column: string) => Promise<{
             key: string;
             value: unknown;
+            time: number;
         }[]>;
-        getSingleField: (key: string, column: string) => Promise<unknown | undefined>;
+        getSingleField: (key: string, column: string) => Promise<{
+            value: unknown;
+            time: number;
+        } | typeof ABSENT>;
     };
     export declare function loadBulkDatabase(config: {
         totalBytes: number;
@@ -911,6 +1007,33 @@ declare module "sliftutils/storage/BulkDatabase2/blockCache" {
 
 }
 
+declare module "sliftutils/storage/BulkDatabase2/manifest" {
+    export declare const MANIFEST_EXTENSION = ".manifest";
+    export type Manifest = {
+        startTime: number;
+        validBulkFiles: string[];
+        ignoredStreamFiles: string[];
+        readFiles: string[];
+    };
+    export declare function isManifestName(name: string): boolean;
+    export declare function manifestFileName(startTime: number, writerId: string, counter: number): string;
+    export declare function parseManifestStartTime(name: string): number | undefined;
+    export declare function chooseManifest(manifests: {
+        name: string;
+        manifest: Manifest;
+    }[]): {
+        name: string;
+        manifest: Manifest;
+    } | undefined;
+
+}
+
+declare module "sliftutils/storage/BulkDatabase2/mergeLock" {
+    export declare function tryAcquireMergeLock(collection: string, holderId: string): boolean;
+    export declare function releaseMergeLock(collection: string, holderId: string): void;
+
+}
+
 declare module "sliftutils/storage/BulkDatabase2/streamLog" {
     /// <reference types="node" />
     /// <reference types="node" />

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.3.0",
+    "version": "1.3.1",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabase2.d.ts

@@ -1,8 +1,77 @@
 import { BulkDatabaseBase } from "./BulkDatabaseBase";
-export { BulkDatabaseBase, noopReactiveDeps } from "./BulkDatabaseBase";
+export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
 export type { ReactiveDeps, StorageFactory } from "./BulkDatabaseBase";
+/** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
+export type BulkColumnInfo = {
+    column: string;
+    byteSize: number;
+};
+/** A snapshot of the collection's shape (no row data), as reported by getReaderInfo. */
+export type BulkReaderInfo = {
+    rowCount: number;
+    totalBytes: number;
+    keyCount: number;
+    sampleKey: string | undefined;
+    columns: BulkColumnInfo[];
+};
+/**
+ * The full public API of BulkDatabase2 (the static `clearCache()` aside). BulkDatabase2 implements
+ * this; an application that just wants to depend on the surface can type against this interface instead
+ * of reading the implementation. `T` is the row type and must have a string `key`.
+ *
+ * Reads resolve every key/column by the latest write-time across all storage tiers. Writes are
+ * column-merges: a write/update only changes the columns it includes; columns it omits keep their
+ * previous value (clear a column by writing it as `undefined`).
+ */
+export interface IBulkDatabase2<T extends {
+    key: string;
+}> {
+    /** The collection name (its folder under the storage root). */
+    readonly name: string;
+    /** Write one full row (merging its columns onto any existing row for the key). */
+    write(entry: T): Promise<void>;
+    /** Write many full rows in one batch. */
+    writeBatch(entries: T[]): Promise<void>;
+    /** Update only the given columns of an existing key (key required). Warns and no-ops if the key isn't present. */
+    update(entry: Partial<T> & {
+        key: string;
+    }): Promise<void>;
+    /** Update many keys' partial columns in one batch. */
+    updateBatch(entries: (Partial<T> & {
+        key: string;
+    })[]): Promise<void>;
+    /** Delete a key. */
+    delete(key: string): Promise<void>;
+    /** Delete many keys in one batch. */
+    deleteBatch(keys: string[]): Promise<void>;
+    /** All live keys. */
+    getKeys(): Promise<string[]>;
+    /** One field's value for a key, or undefined if the key/column isn't set or the key is deleted. */
+    getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
+    /** A whole column as {key, value} for every live key. */
+    getColumn<Column extends keyof T>(column: Column): Promise<{
+        key: string;
+        value: T[Column];
+    }[]>;
+    /**
+     * Synchronous, reactive read of one field. Returns undefined while the base value is still loading
+     * (and re-renders once it arrives, under a mobx observer); reflects pending writes immediately.
+     */
+    getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
+    /** Synchronous, reactive read of a whole column. undefined while still loading. */
+    getColumnSync<Column extends keyof T>(column: Column): {
+        key: string;
+        value: T[Column];
+    }[] | undefined;
+    /** The columns present on disk and their byte sizes (no row data read). */
+    getColumnInfo(): Promise<BulkColumnInfo[]>;
+    /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
+    getReaderInfo(): Promise<BulkReaderInfo>;
+    /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
+    compact(): Promise<void>;
+}
 export declare class BulkDatabase2<T extends {
     key: string;
-}> extends BulkDatabaseBase<T> {
+}> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {
     constructor(name: string);
 }

package/storage/BulkDatabase2/BulkDatabase2.ts

@@ -2,9 +2,71 @@ import { getFileStorageNested2 } from "../FileFolderAPI";
 import { observable, runInAction } from "../../render-utils/mobxTyped";
 import { BulkDatabaseBase, ReactiveDeps } from "./BulkDatabaseBase";
 
-export { BulkDatabaseBase, noopReactiveDeps } from "./BulkDatabaseBase";
+export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
 export type { ReactiveDeps, StorageFactory } from "./BulkDatabaseBase";
 
+/** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
+export type BulkColumnInfo = { column: string; byteSize: number };
+
+/** A snapshot of the collection's shape (no row data), as reported by getReaderInfo. */
+export type BulkReaderInfo = {
+    rowCount: number;
+    totalBytes: number;
+    keyCount: number;
+    sampleKey: string | undefined;
+    columns: BulkColumnInfo[];
+};
+
+/**
+ * The full public API of BulkDatabase2 (the static `clearCache()` aside). BulkDatabase2 implements
+ * this; an application that just wants to depend on the surface can type against this interface instead
+ * of reading the implementation. `T` is the row type and must have a string `key`.
+ *
+ * Reads resolve every key/column by the latest write-time across all storage tiers. Writes are
+ * column-merges: a write/update only changes the columns it includes; columns it omits keep their
+ * previous value (clear a column by writing it as `undefined`).
+ */
+export interface IBulkDatabase2<T extends { key: string }> {
+    /** The collection name (its folder under the storage root). */
+    readonly name: string;
+
+    /** Write one full row (merging its columns onto any existing row for the key). */
+    write(entry: T): Promise<void>;
+    /** Write many full rows in one batch. */
+    writeBatch(entries: T[]): Promise<void>;
+    /** Update only the given columns of an existing key (key required). Warns and no-ops if the key isn't present. */
+    update(entry: Partial<T> & { key: string }): Promise<void>;
+    /** Update many keys' partial columns in one batch. */
+    updateBatch(entries: (Partial<T> & { key: string })[]): Promise<void>;
+    /** Delete a key. */
+    delete(key: string): Promise<void>;
+    /** Delete many keys in one batch. */
+    deleteBatch(keys: string[]): Promise<void>;
+
+    /** All live keys. */
+    getKeys(): Promise<string[]>;
+    /** One field's value for a key, or undefined if the key/column isn't set or the key is deleted. */
+    getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
+    /** A whole column as {key, value} for every live key. */
+    getColumn<Column extends keyof T>(column: Column): Promise<{ key: string; value: T[Column] }[]>;
+
+    /**
+     * Synchronous, reactive read of one field. Returns undefined while the base value is still loading
+     * (and re-renders once it arrives, under a mobx observer); reflects pending writes immediately.
+     */
+    getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
+    /** Synchronous, reactive read of a whole column. undefined while still loading. */
+    getColumnSync<Column extends keyof T>(column: Column): { key: string; value: T[Column] }[] | undefined;
+
+    /** The columns present on disk and their byte sizes (no row data read). */
+    getColumnInfo(): Promise<BulkColumnInfo[]>;
+    /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
+    getReaderInfo(): Promise<BulkReaderInfo>;
+
+    /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
+    compact(): Promise<void>;
+}
+
 // mobx-backed reactivity: each signal string gets its own observable box. observe() reads it (so an
 // observer/autorun that calls it tracks that box) and invalidate() bumps it (so those reactions re-run).
 // This reproduces the fine-grained per-key + load-version reactivity the class had when it used
@@ -34,7 +96,7 @@ class MobxReactiveDeps implements ReactiveDeps {
 // Backwards-compatible BulkDatabase2: the mobx-reactive flavor. All behavior lives in BulkDatabaseBase;
 // this just supplies mobx reactivity and the default (getFileStorageNested2) storage backend, so the
 // sync reads (getSingleFieldSync / getColumnSync) stay observable for mobx components.
-export class BulkDatabase2<T extends { key: string }> extends BulkDatabaseBase<T> {
+export class BulkDatabase2<T extends { key: string }> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {
     constructor(name: string) {
         super(name, new MobxReactiveDeps(), getFileStorageNested2);
     }

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -1,4 +1,12 @@
 import type { FileStorage } from "../FileFolderAPI";
+export declare const bulkDatabase2Timing: {
+    streamSealAgeMs: number;
+    foldDataAgeMs: number;
+    foldTriggerAgeMs: number;
+    foldCheckIntervalMs: number;
+    cleanupAgeMs: number;
+    cleanupIntervalMs: number;
+};
 export interface ReactiveDeps {
     observe(signal: string): void;
     invalidate(signal: string): void;
@@ -22,9 +30,12 @@ export declare class BulkDatabaseBase<T extends {
     private overlay;
     private streamTimes;
     private streamFileName;
-    private streamRowsWritten;
+    private lastCleanup;
+    private lastFoldCheck;
     private getStreamFileName;
-    private setOverlay;
+    private invalidateOverlay;
+    private setOverlayRow;
+    private setOverlayDeleted;
     private reader;
     private syncSetup;
     private localTime;
@@ -34,20 +45,28 @@ export declare class BulkDatabaseBase<T extends {
     writeBatch(entries: T[]): Promise<void>;
     delete(key: string): Promise<void>;
     deleteBatch(keys: string[]): Promise<void>;
+    update(entry: Partial<T> & {
+        key: string;
+    }): Promise<void>;
+    updateBatch(entries: (Partial<T> & {
+        key: string;
+    })[]): Promise<void>;
+    private getValidFiles;
+    private commitManifest;
     private writeBulkFile;
-    private listStreamFiles;
+    private consolidate;
     private loadStreamEntries;
     private orderStreamEntries;
     private maybeRolloverStream;
-    private rolloverStream;
     compact(): Promise<void>;
-    private listFiles;
     private makeRawGetRange;
     private loadFileReader;
     private fileLogicalSize;
     private handleUnreadableFile;
     private mergeFilesBase;
     private mergeFiles;
+    private mergeFilesLocked;
+    private cleanup;
     private formatInfo;
     private patchColumn;
     getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;