sliftutils 1.4.79 → 1.4.81

package/index.d.ts

@@ -963,8 +963,11 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private storageFactory;
         private config;
         constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory, config?: BulkDatabase2Config);
+        private _reader;
+        private get reader();
+        private activated;
+        private activate;
         private setupVisibilityMergeCheck;
-        private reader;
         private subCaches;
         private pendingAppends;
         private flushTimer;
@@ -2282,6 +2285,11 @@ declare module "sliftutils/storage/embeddingFormats" {
     }): StoredEmbedding;
     export declare function serializeStoredEmbedding(stored: StoredEmbedding): string;
     export declare function deserializeStoredEmbedding(base64: string): StoredEmbedding;
+    export declare function averageEmbeddings(embeddings: StoredEmbedding[], config: {
+        format: EmbeddingFormat;
+        model: string;
+    }): StoredEmbedding;
+    export declare function hashEmbedding(stored: StoredEmbedding): string;
     export declare const getCloseness: (embedding1: Float32Array | StoredEmbedding, embedding2: Float32Array | StoredEmbedding) => number;
 
 }
@@ -2311,12 +2319,48 @@ declare module "sliftutils/storage/proxydatabase/Database" {
 
 }
 
+declare module "sliftutils/storage/proxydatabase/ivfEmbeddingDatabase" {
+    import { Database } from "./Database";
+    import { TransactionSetStore } from "./transactionSet";
+    import { StoredEmbedding, EmbeddingFormat } from "../embeddingFormats";
+    export type IvfConfig = {
+        model: string;
+        format: EmbeddingFormat;
+        cellTarget: number;
+        splitAt: number;
+    };
+    export type IvfEmbeddingRoot = {
+        config: IvfConfig;
+        centroids: TransactionSetStore;
+        cells: {
+            [cellId: string]: TransactionSetStore;
+        };
+        refIndex: TransactionSetStore;
+    };
+    export type EmbeddingInput = {
+        ref: string;
+        embedding: StoredEmbedding;
+    };
+    export type SearchHit = {
+        ref: string;
+        closeness: number;
+    };
+    export declare function searchEmbeddings(database: Database<IvfEmbeddingRoot>, query: StoredEmbedding, options: {
+        probeBudget: number;
+        resultCount: number;
+    }): SearchHit[] | undefined;
+    export declare function insertEmbeddings(database: Database<IvfEmbeddingRoot>, items: EmbeddingInput[]): true | undefined;
+    export declare function removeEmbeddings(database: Database<IvfEmbeddingRoot>, refs: string[]): true | undefined;
+
+}
+
 declare module "sliftutils/storage/proxydatabase/transactionSet" {
     import { Database } from "./Database";
     export type TransactionSetStore = {
         [fileNumber: string]: Uint8Array;
     };
     export declare function transactionRead<Value>(database: Database<TransactionSetStore>): Map<string, Value> | undefined;
+    export declare function replayTransactionStore<Value>(store: TransactionSetStore | undefined): Map<string, Value>;
     export declare function transactionMutate<Value>(database: Database<TransactionSetStore>, transactions: {
         key: string;
         value: Value | undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.79",
+    "version": "1.4.81",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/spec.txt

@@ -1,5 +1,42 @@
 
+Searching
+    1) We want to create something that's database agnostic
+        - So I think we need like multiple phases for reading. And for applying it? We need to be able to know all of our reads ahead of time, which we should be able to roughly know. Or at least we'll have multiple stages. That should be fine. I guess we could do it so it actually understands our database. Like it reads, and then if it's not there, it returns. That might be fine too. 
 
+interface Database<Root> {
+    readData: <T>(fnc: (root: Root) => T) => T | undefined;
+    writeData: <T>(fnc: (root: Root) => T, newValue: T) => void;
+}
+I guess this works. We need multiple stages, so it's gonna have to read in the index, In parallel, and then once it has the full index, then it can do the next step in parallel, etc. 
 
 
-Create a companion large data storage thing.  which updates the data less frequently, Where we reference it with unique IDs so we don't have to merge as much and it doesn't have to be as strict. This should allow our large databases to be merged faster, even though they are pretty fast to merge at the moment, but they probably will get slower if we make them 10 times bigger. 
+remaining
+    1) The code book creation. We kind of do want to recreate it sometimes. Maybe we can have something that does a full expensive recreation at certain early on points?
+    2) The searching there's a few different stages. Like, we want to search more IVF cells, but then even within the cells, we kind of want to download more of the final embeddings.  because the codes aren't gonna to be perfect. So we have to do both at the same time?
+
+parts
+    p8g8 format (q8), so we can store 2048 dimensions is a bit over 2KB
+    IVF cell size 32? PQ index reduces by 32X, and if IVF cells reduce by 32X... then 500K 2KB values (1GB) would still just be... 1MB IVF index, so... that's good.
+    0) In IVF, sample 32, then keep sampling more, up to 128 cells
+    1) Pq 64 index (byte values each, break into 64 groups, always... should work well, For face vectors, they're already really optimized. They have so few dimensions, and so we don't compress it by that much. But for text vectors, they're less compressed, so we compress it more. So it should work in all cases)
+    2) Retraining index on a subset
+        - Or... just don't. Freeze after a certain point, and... it might be fine? HMM... Although no, we could allow retraining. It should be cheap enough. 
+    3) Split into transaction logs, so we can read the PQ index efficiently, but still update it
+        UGH... I mean basically you would just be replicating some of what we actually store on disk, but in our database, which is stupid, but I think we'll probably do it anyway. Basically, just a transaction log. We have to read everything every time, but it should help the speed a little bit...
+            - And I think it's fair because it's us saying, hey, we're going to read all this data every time, which the database can't know, so it can't optimize for that, but we can know, and then we can optimize for it
+    4) IVF the pq index to vastly decrease the initial access time
+    5.0) automatically split cells when they get big enough (2X their initial size)
+    5) occasionally regenerate the entire IVF.
+    5.1) Have something to store a subset of the data codes?
+    5.1) Have something that stores some of the data randomly so we can easily access a subset. We won't represent deletions in it. That's fine. This will allow us to rebalance the IVF fairly efficiently. 
+        - And maybe even slowly move stuff over to the new IVF as we access stuff, we see if it's in the new IVF, and if not, we move it?
+        - 10% of the data? And... maybe just the pq index values?
+        - Should make retraining the IVF ten times faster and shouldn't affect the recall by too much, especially if we have more cells
+        - AND, Maybe we can actually sample less the more data we have. Ramping up to 10%. 
+unknown
+    exactly how we're going to update the IVF. 
+    IVF pq index, so we can read part of it
+        mini transaction logs in each cell?
+
+    BUT... how do we update IVF
+

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -35,8 +35,11 @@ export declare class BulkDatabaseBase<T extends {
     private storageFactory;
     private config;
     constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory, config?: BulkDatabase2Config);
+    private _reader;
+    private get reader();
+    private activated;
+    private activate;
     private setupVisibilityMergeCheck;
-    private reader;
     private subCaches;
     private pendingAppends;
     private flushTimer;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -148,14 +148,31 @@ export class BulkDatabaseBase<T extends { key: string }> {
         protected deps: ReactiveDeps,
         private storageFactory: StorageFactory,
         private config: BulkDatabase2Config = {},
-    ) {
-        this.reader = new BulkDatabaseReader<T>({
-            name,
-            deps,
-            maxTriggerThrottleMs: config.maxTriggerThrottleMs,
+    ) { }
+
+    // The reader (and the background machinery that rides along with it) must NOT be set up just
+    // because the collection was constructed. Many collections are constructed and never touched, and
+    // in Node a merge tick would poke the storage factory (e.g. indexedDB) and throw. We build it
+    // lazily on first access to `this.reader`: every read, write, and merge goes through the reader,
+    // while pure construction never touches it.
+    private _reader: BulkDatabaseReader<T> | undefined;
+    private get reader(): BulkDatabaseReader<T> {
+        if (this._reader) return this._reader;
+        const reader = new BulkDatabaseReader<T>({
+            name: this.name,
+            deps: this.deps,
+            maxTriggerThrottleMs: this.config.maxTriggerThrottleMs,
         });
-        this.reader.setEnsureIndex(() => this.ensureIndex());
+        reader.setEnsureIndex(() => this.ensureIndex());
+        this._reader = reader;
+        this.activate();
+        return reader;
+    }
 
+    private activated = false;
+    private activate(): void {
+        if (this.activated) return;
+        this.activated = true;
         if (typeof window !== "undefined") {
             try {
                 window.addEventListener("pagehide", () => void this.flushPending());
@@ -205,7 +222,6 @@ export class BulkDatabaseBase<T extends { key: string }> {
         } catch { /* not in a DOM context */ }
     }
 
-    private reader: BulkDatabaseReader<T>;
     private subCaches: SubReaderCaches = { bulk: new Map(), stream: new Map() };
 
     private pendingAppends: { framed: Buffer; rows: number }[] = [];

package/storage/embeddingFormats.d.ts

@@ -21,4 +21,9 @@ export declare function encodeEmbedding(config: {
 }): StoredEmbedding;
 export declare function serializeStoredEmbedding(stored: StoredEmbedding): string;
 export declare function deserializeStoredEmbedding(base64: string): StoredEmbedding;
+export declare function averageEmbeddings(embeddings: StoredEmbedding[], config: {
+    format: EmbeddingFormat;
+    model: string;
+}): StoredEmbedding;
+export declare function hashEmbedding(stored: StoredEmbedding): string;
 export declare const getCloseness: (embedding1: Float32Array | StoredEmbedding, embedding2: Float32Array | StoredEmbedding) => number;

package/storage/embeddingFormats.ts

@@ -273,6 +273,39 @@ export function deserializeStoredEmbedding(base64: string): StoredEmbedding {
     };
 }
 
+// Mean of several embeddings (decoded to their common length), re-encoded into the requested format
+// (which renormalizes). Used to build and split IVF cell centroids.
+export function averageEmbeddings(embeddings: StoredEmbedding[], config: { format: EmbeddingFormat; model: string }): StoredEmbedding {
+    let length = Infinity;
+    for (let embedding of embeddings) {
+        length = Math.min(length, embeddingLength(embedding));
+    }
+    let sum = new Float32Array(length);
+    for (let embedding of embeddings) {
+        let decoded = decodeToLength(embedding, length);
+        for (let dimension = 0; dimension < length; dimension++) {
+            sum[dimension] += decoded[dimension];
+        }
+    }
+    for (let dimension = 0; dimension < length; dimension++) {
+        sum[dimension] /= embeddings.length;
+    }
+    return encodeEmbedding({ input: sum, format: config.format, model: config.model });
+}
+
+// A stable 16-byte (base64) content hash of an embedding, for use as a cell id derived from its centroid.
+export function hashEmbedding(stored: StoredEmbedding): string {
+    let serialized = serializeStoredEmbedding(stored);
+    let lanes = new Uint32Array([2166136261, 2654435761, 40503, 3266489917]);
+    for (let charIndex = 0; charIndex < serialized.length; charIndex++) {
+        let code = serialized.charCodeAt(charIndex);
+        for (let lane = 0; lane < lanes.length; lane++) {
+            lanes[lane] = Math.imul(lanes[lane] ^ (code + lane * 131 + charIndex), 16777619);
+        }
+    }
+    return Buffer.from(lanes.buffer).toString("base64");
+}
+
 // Compare two embeddings in any format / truncation. Both are decoded, truncated to their common
 // length, and unit-normalized, then scored as 1 minus their euclidean distance.
 export const getCloseness = measureWrap(function getCloseness(

package/storage/proxydatabase/ivfEmbeddingDatabase.d.ts

@@ -0,0 +1,31 @@
+import { Database } from "./Database";
+import { TransactionSetStore } from "./transactionSet";
+import { StoredEmbedding, EmbeddingFormat } from "../embeddingFormats";
+export type IvfConfig = {
+    model: string;
+    format: EmbeddingFormat;
+    cellTarget: number;
+    splitAt: number;
+};
+export type IvfEmbeddingRoot = {
+    config: IvfConfig;
+    centroids: TransactionSetStore;
+    cells: {
+        [cellId: string]: TransactionSetStore;
+    };
+    refIndex: TransactionSetStore;
+};
+export type EmbeddingInput = {
+    ref: string;
+    embedding: StoredEmbedding;
+};
+export type SearchHit = {
+    ref: string;
+    closeness: number;
+};
+export declare function searchEmbeddings(database: Database<IvfEmbeddingRoot>, query: StoredEmbedding, options: {
+    probeBudget: number;
+    resultCount: number;
+}): SearchHit[] | undefined;
+export declare function insertEmbeddings(database: Database<IvfEmbeddingRoot>, items: EmbeddingInput[]): true | undefined;
+export declare function removeEmbeddings(database: Database<IvfEmbeddingRoot>, refs: string[]): true | undefined;

package/storage/proxydatabase/ivfEmbeddingDatabase.ts

@@ -0,0 +1,227 @@
+import { Database, namespaceDatabase } from "./Database";
+import { TransactionSetStore, transactionRead, transactionMutate, replayTransactionStore } from "./transactionSet";
+import { StoredEmbedding, EmbeddingFormat, getCloseness, averageEmbeddings, hashEmbedding } from "../embeddingFormats";
+
+export type IvfConfig = {
+    model: string;
+    format: EmbeddingFormat;
+    cellTarget: number;
+    splitAt: number;
+};
+
+// Cell ids ARE the hash of the cell's centroid, so there's no id counter. centroids maps that id to the
+// centroid embedding (read whole as the preload, ranked in RAM). Each cell is its own transaction set of
+// ref → member embedding. refIndex maps a ref to the cell holding it, so a remove can find it.
+export type IvfEmbeddingRoot = {
+    config: IvfConfig;
+    centroids: TransactionSetStore;
+    cells: { [cellId: string]: TransactionSetStore };
+    refIndex: TransactionSetStore;
+};
+
+export type EmbeddingInput = { ref: string; embedding: StoredEmbedding };
+export type SearchHit = { ref: string; closeness: number };
+
+function centroidStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore> {
+    return namespaceDatabase(database, root => root.centroids);
+}
+function cellStore(database: Database<IvfEmbeddingRoot>, cellId: string): Database<TransactionSetStore> {
+    return namespaceDatabase(database, root => root.cells[cellId]);
+}
+function refIndexStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore> {
+    return namespaceDatabase(database, root => root.refIndex);
+}
+
+function nearestCell(embedding: StoredEmbedding, centroids: Map<string, StoredEmbedding>): string {
+    let bestCellId = "";
+    let bestCloseness = -Infinity;
+    for (const [cellId, centroid] of centroids) {
+        const closeness = getCloseness(embedding, centroid);
+        if (closeness > bestCloseness) {
+            bestCloseness = closeness;
+            bestCellId = cellId;
+        }
+    }
+    return bestCellId;
+}
+
+// 2-means on a cell's members so an overgrown cell can split in place. Seeds with one member and the one
+// farthest from it, then alternates assign / recompute-centroid a few times.
+function splitInTwo(
+    members: [string, StoredEmbedding][],
+    config: IvfConfig,
+): { centroid: StoredEmbedding; members: [string, StoredEmbedding][] }[] {
+    let centroidA = members[0][1];
+    let centroidB = members[0][1];
+    let worst = Infinity;
+    for (const [, embedding] of members) {
+        const closeness = getCloseness(centroidA, embedding);
+        if (closeness < worst) {
+            worst = closeness;
+            centroidB = embedding;
+        }
+    }
+    let groupA: [string, StoredEmbedding][] = [];
+    let groupB: [string, StoredEmbedding][] = [];
+    for (let iteration = 0; iteration < 5; iteration++) {
+        groupA = [];
+        groupB = [];
+        for (const member of members) {
+            if (getCloseness(centroidA, member[1]) >= getCloseness(centroidB, member[1])) {
+                groupA.push(member);
+            } else {
+                groupB.push(member);
+            }
+        }
+        if (!groupA.length || !groupB.length) {
+            break;
+        }
+        centroidA = averageEmbeddings(groupA.map(member => member[1]), config);
+        centroidB = averageEmbeddings(groupB.map(member => member[1]), config);
+    }
+    const result: { centroid: StoredEmbedding; members: [string, StoredEmbedding][] }[] = [];
+    if (groupA.length) {
+        result.push({ centroid: centroidA, members: groupA });
+    }
+    if (groupB.length) {
+        result.push({ centroid: centroidB, members: groupB });
+    }
+    return result;
+}
+
+// Nearest cells until probeBudget members are covered, scored exactly against the query. undefined while
+// not synced. All probed cells are fetched in one batched read so probing doesn't cascade per cell.
+export function searchEmbeddings(
+    database: Database<IvfEmbeddingRoot>,
+    query: StoredEmbedding,
+    options: { probeBudget: number; resultCount: number },
+): SearchHit[] | undefined {
+    const config = database.readData(root => root.config);
+    if (!config) return undefined;
+    const centroids = transactionRead<StoredEmbedding>(centroidStore(database));
+    if (!centroids) return undefined;
+    if (!centroids.size) return [];
+
+    const rankedCellIds = [...centroids.entries()]
+        .map(([cellId, centroid]) => ({ cellId, closeness: getCloseness(query, centroid) }))
+        .sort((left, right) => right.closeness - left.closeness)
+        .map(ranked => ranked.cellId);
+
+    const probeCellCount = Math.max(1, Math.ceil(options.probeBudget / config.cellTarget));
+    const probeCellIds = rankedCellIds.slice(0, probeCellCount);
+    const stores = database.readData(root => probeCellIds.map(cellId => root.cells[cellId]));
+    if (!stores) return undefined;
+
+    const hits: SearchHit[] = [];
+    for (const store of stores) {
+        const members = replayTransactionStore<StoredEmbedding>(store);
+        for (const [ref, embedding] of members) {
+            hits.push({ ref, closeness: getCloseness(query, embedding) });
+        }
+    }
+    hits.sort((left, right) => right.closeness - left.closeness);
+    return hits.slice(0, options.resultCount);
+}
+
+// Assign each embedding to its nearest cell and add it. An empty index bootstraps a first cell from the
+// inserted batch; a cell that crosses splitAt*cellTarget splits in place. undefined while not synced.
+export function insertEmbeddings(
+    database: Database<IvfEmbeddingRoot>,
+    items: EmbeddingInput[],
+): true | undefined {
+    if (!items.length) return true;
+    const config = database.readData(root => root.config);
+    if (!config) return undefined;
+    const centroids = transactionRead<StoredEmbedding>(centroidStore(database));
+    if (!centroids) return undefined;
+
+    const newCentroids: { key: string; value: StoredEmbedding | undefined }[] = [];
+    if (!centroids.size) {
+        const centroid = averageEmbeddings(items.map(item => item.embedding), config);
+        const cellId = hashEmbedding(centroid);
+        centroids.set(cellId, centroid);
+        newCentroids.push({ key: cellId, value: centroid });
+    }
+
+    const itemsByCell = new Map<string, EmbeddingInput[]>();
+    for (const item of items) {
+        const cellId = nearestCell(item.embedding, centroids);
+        let group = itemsByCell.get(cellId);
+        if (!group) {
+            group = [];
+            itemsByCell.set(cellId, group);
+        }
+        group.push(item);
+    }
+
+    const targetCellIds = [...itemsByCell.keys()];
+    const existingStores = database.readData(root => targetCellIds.map(cellId => root.cells[cellId]));
+    if (!existingStores) return undefined;
+
+    const splitThreshold = config.splitAt * config.cellTarget;
+    const refWrites: { key: string; value: string | undefined }[] = [];
+    for (let cellIndex = 0; cellIndex < targetCellIds.length; cellIndex++) {
+        const cellId = targetCellIds[cellIndex];
+        const group = itemsByCell.get(cellId)!;
+        const combined = replayTransactionStore<StoredEmbedding>(existingStores[cellIndex]);
+        for (const item of group) {
+            combined.set(item.ref, item.embedding);
+        }
+
+        if (combined.size <= splitThreshold) {
+            const memberWrites = group.map(item => ({ key: item.ref, value: item.embedding }));
+            if (!transactionMutate(cellStore(database, cellId), memberWrites)) return undefined;
+            for (const item of group) {
+                refWrites.push({ key: item.ref, value: cellId });
+            }
+            continue;
+        }
+
+        const subCells = splitInTwo([...combined.entries()], config);
+        for (const subCell of subCells) {
+            const subCellId = hashEmbedding(subCell.centroid);
+            newCentroids.push({ key: subCellId, value: subCell.centroid });
+            const memberWrites = subCell.members.map(([ref, embedding]) => ({ key: ref, value: embedding }));
+            if (!transactionMutate(cellStore(database, subCellId), memberWrites)) return undefined;
+            for (const [ref] of subCell.members) {
+                refWrites.push({ key: ref, value: subCellId });
+            }
+        }
+        newCentroids.push({ key: cellId, value: undefined });
+        database.deleteData(root => root.cells[cellId]);
+    }
+
+    if (newCentroids.length && !transactionMutate(centroidStore(database), newCentroids)) return undefined;
+    if (refWrites.length && !transactionMutate(refIndexStore(database), refWrites)) return undefined;
+    return true;
+}
+
+// Tombstone each ref in its cell (found via refIndex) and drop it from the index. undefined while not synced.
+export function removeEmbeddings(
+    database: Database<IvfEmbeddingRoot>,
+    refs: string[],
+): true | undefined {
+    if (!refs.length) return true;
+    const refIndex = transactionRead<string>(refIndexStore(database));
+    if (!refIndex) return undefined;
+
+    const refsByCell = new Map<string, string[]>();
+    for (const ref of refs) {
+        const cellId = refIndex.get(ref);
+        if (!cellId) continue;
+        let group = refsByCell.get(cellId);
+        if (!group) {
+            group = [];
+            refsByCell.set(cellId, group);
+        }
+        group.push(ref);
+    }
+
+    for (const [cellId, group] of refsByCell) {
+        const memberDeletes = group.map(ref => ({ key: ref, value: undefined }));
+        if (!transactionMutate<StoredEmbedding>(cellStore(database, cellId), memberDeletes)) return undefined;
+    }
+    const refDeletes = refs.map(ref => ({ key: ref, value: undefined }));
+    if (!transactionMutate<string>(refIndexStore(database), refDeletes)) return undefined;
+    return true;
+}

package/storage/proxydatabase/transactionSet.d.ts

@@ -3,6 +3,7 @@ export type TransactionSetStore = {
     [fileNumber: string]: Uint8Array;
 };
 export declare function transactionRead<Value>(database: Database<TransactionSetStore>): Map<string, Value> | undefined;
+export declare function replayTransactionStore<Value>(store: TransactionSetStore | undefined): Map<string, Value>;
 export declare function transactionMutate<Value>(database: Database<TransactionSetStore>, transactions: {
     key: string;
     value: Value | undefined;

package/storage/proxydatabase/transactionSet.ts

@@ -21,6 +21,17 @@ function maxNumber(values: number[], fallback: number): number {
     return result;
 }
 
+function decodeBuffers(buffers: Uint8Array[]): Transaction[] {
+    const transactions: Transaction[] = [];
+    for (const buffer of buffers) {
+        const decoded = transactionCbor.decode(buffer) as Transaction[];
+        for (const transaction of decoded) {
+            transactions.push(transaction);
+        }
+    }
+    return transactions;
+}
+
 function readTransactionFiles(
     database: Database<TransactionSetStore>,
 ): { fileKeys: string[]; transactions: Transaction[] } | undefined {
@@ -29,14 +40,7 @@ function readTransactionFiles(
         buffers: Object.values(store),
     }));
     if (!snapshot) return undefined;
-    const transactions: Transaction[] = [];
-    for (const buffer of snapshot.buffers) {
-        const decoded = transactionCbor.decode(buffer) as Transaction[];
-        for (const transaction of decoded) {
-            transactions.push(transaction);
-        }
-    }
-    return { fileKeys: snapshot.fileKeys, transactions };
+    return { fileKeys: snapshot.fileKeys, transactions: decodeBuffers(snapshot.buffers) };
 }
 
 function replay(transactions: Transaction[]): Map<string, unknown> {
@@ -77,6 +81,12 @@ export function transactionRead<Value>(
     return replay(files.transactions) as Map<string, Value>;
 }
 
+// Replay an already-read store object (no read of its own), so a caller can fetch many sets in one batched read and then materialize each.
+export function replayTransactionStore<Value>(store: TransactionSetStore | undefined): Map<string, Value> {
+    if (!store) return new Map();
+    return replay(decodeBuffers(Object.values(store))) as Map<string, Value>;
+}
+
 export function transactionMutate<Value>(
     database: Database<TransactionSetStore>,
     transactions: { key: string; value: Value | undefined }[],

package/storage/remoteFileServer.ts

@@ -12,6 +12,13 @@ import { WebSocketServer, WebSocket } from "ws";
 import { BulkDatabaseBase, bulkDatabase2Timing, noopReactiveDeps, BULK_ROOT_FOLDER } from "./BulkDatabase2/BulkDatabaseBase";
 import { wrapHandle, NodeJSDirectoryHandleWrapper, DirectoryWrapper } from "./FileFolderAPI";
 
+process.on("uncaughtException", err => {
+    console.error(`[writeServer] uncaughtException (continuing):`, (err as Error).stack ?? err);
+});
+process.on("unhandledRejection", reason => {
+    console.error(`[writeServer] unhandledRejection (continuing):`, (reason as Error)?.stack ?? reason);
+});
+
 // Remote file server for sliftutils getRemoteFileStorage / BulkDatabase2. Serves one folder on disk over
 // self-signed HTTPS, authenticated with an auto-generated 6-word password (sent as
 // `Authorization: Bearer <password>`). Run it with `yarn filehoster <folder>` (or the `filehoster` bin).