sliftutils 1.4.80 → 1.4.81

package/index.d.ts

@@ -963,8 +963,11 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private storageFactory;
         private config;
         constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory, config?: BulkDatabase2Config);
+        private _reader;
+        private get reader();
+        private activated;
+        private activate;
         private setupVisibilityMergeCheck;
-        private reader;
         private subCaches;
         private pendingAppends;
         private flushTimer;
@@ -2282,6 +2285,11 @@ declare module "sliftutils/storage/embeddingFormats" {
     }): StoredEmbedding;
     export declare function serializeStoredEmbedding(stored: StoredEmbedding): string;
     export declare function deserializeStoredEmbedding(base64: string): StoredEmbedding;
+    export declare function averageEmbeddings(embeddings: StoredEmbedding[], config: {
+        format: EmbeddingFormat;
+        model: string;
+    }): StoredEmbedding;
+    export declare function hashEmbedding(stored: StoredEmbedding): string;
     export declare const getCloseness: (embedding1: Float32Array | StoredEmbedding, embedding2: Float32Array | StoredEmbedding) => number;
 
 }
@@ -2311,12 +2319,48 @@ declare module "sliftutils/storage/proxydatabase/Database" {
 
 }
 
+declare module "sliftutils/storage/proxydatabase/ivfEmbeddingDatabase" {
+    import { Database } from "./Database";
+    import { TransactionSetStore } from "./transactionSet";
+    import { StoredEmbedding, EmbeddingFormat } from "../embeddingFormats";
+    export type IvfConfig = {
+        model: string;
+        format: EmbeddingFormat;
+        cellTarget: number;
+        splitAt: number;
+    };
+    export type IvfEmbeddingRoot = {
+        config: IvfConfig;
+        centroids: TransactionSetStore;
+        cells: {
+            [cellId: string]: TransactionSetStore;
+        };
+        refIndex: TransactionSetStore;
+    };
+    export type EmbeddingInput = {
+        ref: string;
+        embedding: StoredEmbedding;
+    };
+    export type SearchHit = {
+        ref: string;
+        closeness: number;
+    };
+    export declare function searchEmbeddings(database: Database<IvfEmbeddingRoot>, query: StoredEmbedding, options: {
+        probeBudget: number;
+        resultCount: number;
+    }): SearchHit[] | undefined;
+    export declare function insertEmbeddings(database: Database<IvfEmbeddingRoot>, items: EmbeddingInput[]): true | undefined;
+    export declare function removeEmbeddings(database: Database<IvfEmbeddingRoot>, refs: string[]): true | undefined;
+
+}
+
 declare module "sliftutils/storage/proxydatabase/transactionSet" {
     import { Database } from "./Database";
     export type TransactionSetStore = {
         [fileNumber: string]: Uint8Array;
     };
     export declare function transactionRead<Value>(database: Database<TransactionSetStore>): Map<string, Value> | undefined;
+    export declare function replayTransactionStore<Value>(store: TransactionSetStore | undefined): Map<string, Value>;
     export declare function transactionMutate<Value>(database: Database<TransactionSetStore>, transactions: {
         key: string;
         value: Value | undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.80",
+    "version": "1.4.81",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -35,8 +35,11 @@ export declare class BulkDatabaseBase<T extends {
     private storageFactory;
     private config;
     constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory, config?: BulkDatabase2Config);
+    private _reader;
+    private get reader();
+    private activated;
+    private activate;
     private setupVisibilityMergeCheck;
-    private reader;
     private subCaches;
     private pendingAppends;
     private flushTimer;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -148,14 +148,31 @@ export class BulkDatabaseBase<T extends { key: string }> {
         protected deps: ReactiveDeps,
         private storageFactory: StorageFactory,
         private config: BulkDatabase2Config = {},
-    ) {
-        this.reader = new BulkDatabaseReader<T>({
-            name,
-            deps,
-            maxTriggerThrottleMs: config.maxTriggerThrottleMs,
+    ) { }
+
+    // The reader (and the background machinery that rides along with it) must NOT be set up just
+    // because the collection was constructed. Many collections are constructed and never touched, and
+    // in Node a merge tick would poke the storage factory (e.g. indexedDB) and throw. We build it
+    // lazily on first access to `this.reader`: every read, write, and merge goes through the reader,
+    // while pure construction never touches it.
+    private _reader: BulkDatabaseReader<T> | undefined;
+    private get reader(): BulkDatabaseReader<T> {
+        if (this._reader) return this._reader;
+        const reader = new BulkDatabaseReader<T>({
+            name: this.name,
+            deps: this.deps,
+            maxTriggerThrottleMs: this.config.maxTriggerThrottleMs,
         });
-        this.reader.setEnsureIndex(() => this.ensureIndex());
+        reader.setEnsureIndex(() => this.ensureIndex());
+        this._reader = reader;
+        this.activate();
+        return reader;
+    }
 
+    private activated = false;
+    private activate(): void {
+        if (this.activated) return;
+        this.activated = true;
         if (typeof window !== "undefined") {
             try {
                 window.addEventListener("pagehide", () => void this.flushPending());
@@ -205,7 +222,6 @@ export class BulkDatabaseBase<T extends { key: string }> {
         } catch { /* not in a DOM context */ }
     }
 
-    private reader: BulkDatabaseReader<T>;
     private subCaches: SubReaderCaches = { bulk: new Map(), stream: new Map() };
 
     private pendingAppends: { framed: Buffer; rows: number }[] = [];

package/storage/embeddingFormats.d.ts

@@ -21,4 +21,9 @@ export declare function encodeEmbedding(config: {
 }): StoredEmbedding;
 export declare function serializeStoredEmbedding(stored: StoredEmbedding): string;
 export declare function deserializeStoredEmbedding(base64: string): StoredEmbedding;
+export declare function averageEmbeddings(embeddings: StoredEmbedding[], config: {
+    format: EmbeddingFormat;
+    model: string;
+}): StoredEmbedding;
+export declare function hashEmbedding(stored: StoredEmbedding): string;
 export declare const getCloseness: (embedding1: Float32Array | StoredEmbedding, embedding2: Float32Array | StoredEmbedding) => number;

package/storage/embeddingFormats.ts

@@ -273,6 +273,39 @@ export function deserializeStoredEmbedding(base64: string): StoredEmbedding {
     };
 }
 
+// Mean of several embeddings (decoded to their common length), re-encoded into the requested format
+// (which renormalizes). Used to build and split IVF cell centroids.
+export function averageEmbeddings(embeddings: StoredEmbedding[], config: { format: EmbeddingFormat; model: string }): StoredEmbedding {
+    let length = Infinity;
+    for (let embedding of embeddings) {
+        length = Math.min(length, embeddingLength(embedding));
+    }
+    let sum = new Float32Array(length);
+    for (let embedding of embeddings) {
+        let decoded = decodeToLength(embedding, length);
+        for (let dimension = 0; dimension < length; dimension++) {
+            sum[dimension] += decoded[dimension];
+        }
+    }
+    for (let dimension = 0; dimension < length; dimension++) {
+        sum[dimension] /= embeddings.length;
+    }
+    return encodeEmbedding({ input: sum, format: config.format, model: config.model });
+}
+
+// A stable 16-byte (base64) content hash of an embedding, for use as a cell id derived from its centroid.
+export function hashEmbedding(stored: StoredEmbedding): string {
+    let serialized = serializeStoredEmbedding(stored);
+    let lanes = new Uint32Array([2166136261, 2654435761, 40503, 3266489917]);
+    for (let charIndex = 0; charIndex < serialized.length; charIndex++) {
+        let code = serialized.charCodeAt(charIndex);
+        for (let lane = 0; lane < lanes.length; lane++) {
+            lanes[lane] = Math.imul(lanes[lane] ^ (code + lane * 131 + charIndex), 16777619);
+        }
+    }
+    return Buffer.from(lanes.buffer).toString("base64");
+}
+
 // Compare two embeddings in any format / truncation. Both are decoded, truncated to their common
 // length, and unit-normalized, then scored as 1 minus their euclidean distance.
 export const getCloseness = measureWrap(function getCloseness(

package/storage/proxydatabase/ivfEmbeddingDatabase.d.ts

@@ -0,0 +1,31 @@
+import { Database } from "./Database";
+import { TransactionSetStore } from "./transactionSet";
+import { StoredEmbedding, EmbeddingFormat } from "../embeddingFormats";
+export type IvfConfig = {
+    model: string;
+    format: EmbeddingFormat;
+    cellTarget: number;
+    splitAt: number;
+};
+export type IvfEmbeddingRoot = {
+    config: IvfConfig;
+    centroids: TransactionSetStore;
+    cells: {
+        [cellId: string]: TransactionSetStore;
+    };
+    refIndex: TransactionSetStore;
+};
+export type EmbeddingInput = {
+    ref: string;
+    embedding: StoredEmbedding;
+};
+export type SearchHit = {
+    ref: string;
+    closeness: number;
+};
+export declare function searchEmbeddings(database: Database<IvfEmbeddingRoot>, query: StoredEmbedding, options: {
+    probeBudget: number;
+    resultCount: number;
+}): SearchHit[] | undefined;
+export declare function insertEmbeddings(database: Database<IvfEmbeddingRoot>, items: EmbeddingInput[]): true | undefined;
+export declare function removeEmbeddings(database: Database<IvfEmbeddingRoot>, refs: string[]): true | undefined;

package/storage/proxydatabase/ivfEmbeddingDatabase.ts

@@ -0,0 +1,227 @@
+import { Database, namespaceDatabase } from "./Database";
+import { TransactionSetStore, transactionRead, transactionMutate, replayTransactionStore } from "./transactionSet";
+import { StoredEmbedding, EmbeddingFormat, getCloseness, averageEmbeddings, hashEmbedding } from "../embeddingFormats";
+
+export type IvfConfig = {
+    model: string;
+    format: EmbeddingFormat;
+    cellTarget: number;
+    splitAt: number;
+};
+
+// Cell ids ARE the hash of the cell's centroid, so there's no id counter. centroids maps that id to the
+// centroid embedding (read whole as the preload, ranked in RAM). Each cell is its own transaction set of
+// ref → member embedding. refIndex maps a ref to the cell holding it, so a remove can find it.
+export type IvfEmbeddingRoot = {
+    config: IvfConfig;
+    centroids: TransactionSetStore;
+    cells: { [cellId: string]: TransactionSetStore };
+    refIndex: TransactionSetStore;
+};
+
+export type EmbeddingInput = { ref: string; embedding: StoredEmbedding };
+export type SearchHit = { ref: string; closeness: number };
+
+function centroidStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore> {
+    return namespaceDatabase(database, root => root.centroids);
+}
+function cellStore(database: Database<IvfEmbeddingRoot>, cellId: string): Database<TransactionSetStore> {
+    return namespaceDatabase(database, root => root.cells[cellId]);
+}
+function refIndexStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore> {
+    return namespaceDatabase(database, root => root.refIndex);
+}
+
+function nearestCell(embedding: StoredEmbedding, centroids: Map<string, StoredEmbedding>): string {
+    let bestCellId = "";
+    let bestCloseness = -Infinity;
+    for (const [cellId, centroid] of centroids) {
+        const closeness = getCloseness(embedding, centroid);
+        if (closeness > bestCloseness) {
+            bestCloseness = closeness;
+            bestCellId = cellId;
+        }
+    }
+    return bestCellId;
+}
+
+// 2-means on a cell's members so an overgrown cell can split in place. Seeds with one member and the one
+// farthest from it, then alternates assign / recompute-centroid a few times.
+function splitInTwo(
+    members: [string, StoredEmbedding][],
+    config: IvfConfig,
+): { centroid: StoredEmbedding; members: [string, StoredEmbedding][] }[] {
+    let centroidA = members[0][1];
+    let centroidB = members[0][1];
+    let worst = Infinity;
+    for (const [, embedding] of members) {
+        const closeness = getCloseness(centroidA, embedding);
+        if (closeness < worst) {
+            worst = closeness;
+            centroidB = embedding;
+        }
+    }
+    let groupA: [string, StoredEmbedding][] = [];
+    let groupB: [string, StoredEmbedding][] = [];
+    for (let iteration = 0; iteration < 5; iteration++) {
+        groupA = [];
+        groupB = [];
+        for (const member of members) {
+            if (getCloseness(centroidA, member[1]) >= getCloseness(centroidB, member[1])) {
+                groupA.push(member);
+            } else {
+                groupB.push(member);
+            }
+        }
+        if (!groupA.length || !groupB.length) {
+            break;
+        }
+        centroidA = averageEmbeddings(groupA.map(member => member[1]), config);
+        centroidB = averageEmbeddings(groupB.map(member => member[1]), config);
+    }
+    const result: { centroid: StoredEmbedding; members: [string, StoredEmbedding][] }[] = [];
+    if (groupA.length) {
+        result.push({ centroid: centroidA, members: groupA });
+    }
+    if (groupB.length) {
+        result.push({ centroid: centroidB, members: groupB });
+    }
+    return result;
+}
+
+// Nearest cells until probeBudget members are covered, scored exactly against the query. undefined while
+// not synced. All probed cells are fetched in one batched read so probing doesn't cascade per cell.
+export function searchEmbeddings(
+    database: Database<IvfEmbeddingRoot>,
+    query: StoredEmbedding,
+    options: { probeBudget: number; resultCount: number },
+): SearchHit[] | undefined {
+    const config = database.readData(root => root.config);
+    if (!config) return undefined;
+    const centroids = transactionRead<StoredEmbedding>(centroidStore(database));
+    if (!centroids) return undefined;
+    if (!centroids.size) return [];
+
+    const rankedCellIds = [...centroids.entries()]
+        .map(([cellId, centroid]) => ({ cellId, closeness: getCloseness(query, centroid) }))
+        .sort((left, right) => right.closeness - left.closeness)
+        .map(ranked => ranked.cellId);
+
+    const probeCellCount = Math.max(1, Math.ceil(options.probeBudget / config.cellTarget));
+    const probeCellIds = rankedCellIds.slice(0, probeCellCount);
+    const stores = database.readData(root => probeCellIds.map(cellId => root.cells[cellId]));
+    if (!stores) return undefined;
+
+    const hits: SearchHit[] = [];
+    for (const store of stores) {
+        const members = replayTransactionStore<StoredEmbedding>(store);
+        for (const [ref, embedding] of members) {
+            hits.push({ ref, closeness: getCloseness(query, embedding) });
+        }
+    }
+    hits.sort((left, right) => right.closeness - left.closeness);
+    return hits.slice(0, options.resultCount);
+}
+
+// Assign each embedding to its nearest cell and add it. An empty index bootstraps a first cell from the
+// inserted batch; a cell that crosses splitAt*cellTarget splits in place. undefined while not synced.
+export function insertEmbeddings(
+    database: Database<IvfEmbeddingRoot>,
+    items: EmbeddingInput[],
+): true | undefined {
+    if (!items.length) return true;
+    const config = database.readData(root => root.config);
+    if (!config) return undefined;
+    const centroids = transactionRead<StoredEmbedding>(centroidStore(database));
+    if (!centroids) return undefined;
+
+    const newCentroids: { key: string; value: StoredEmbedding | undefined }[] = [];
+    if (!centroids.size) {
+        const centroid = averageEmbeddings(items.map(item => item.embedding), config);
+        const cellId = hashEmbedding(centroid);
+        centroids.set(cellId, centroid);
+        newCentroids.push({ key: cellId, value: centroid });
+    }
+
+    const itemsByCell = new Map<string, EmbeddingInput[]>();
+    for (const item of items) {
+        const cellId = nearestCell(item.embedding, centroids);
+        let group = itemsByCell.get(cellId);
+        if (!group) {
+            group = [];
+            itemsByCell.set(cellId, group);
+        }
+        group.push(item);
+    }
+
+    const targetCellIds = [...itemsByCell.keys()];
+    const existingStores = database.readData(root => targetCellIds.map(cellId => root.cells[cellId]));
+    if (!existingStores) return undefined;
+
+    const splitThreshold = config.splitAt * config.cellTarget;
+    const refWrites: { key: string; value: string | undefined }[] = [];
+    for (let cellIndex = 0; cellIndex < targetCellIds.length; cellIndex++) {
+        const cellId = targetCellIds[cellIndex];
+        const group = itemsByCell.get(cellId)!;
+        const combined = replayTransactionStore<StoredEmbedding>(existingStores[cellIndex]);
+        for (const item of group) {
+            combined.set(item.ref, item.embedding);
+        }
+
+        if (combined.size <= splitThreshold) {
+            const memberWrites = group.map(item => ({ key: item.ref, value: item.embedding }));
+            if (!transactionMutate(cellStore(database, cellId), memberWrites)) return undefined;
+            for (const item of group) {
+                refWrites.push({ key: item.ref, value: cellId });
+            }
+            continue;
+        }
+
+        const subCells = splitInTwo([...combined.entries()], config);
+        for (const subCell of subCells) {
+            const subCellId = hashEmbedding(subCell.centroid);
+            newCentroids.push({ key: subCellId, value: subCell.centroid });
+            const memberWrites = subCell.members.map(([ref, embedding]) => ({ key: ref, value: embedding }));
+            if (!transactionMutate(cellStore(database, subCellId), memberWrites)) return undefined;
+            for (const [ref] of subCell.members) {
+                refWrites.push({ key: ref, value: subCellId });
+            }
+        }
+        newCentroids.push({ key: cellId, value: undefined });
+        database.deleteData(root => root.cells[cellId]);
+    }
+
+    if (newCentroids.length && !transactionMutate(centroidStore(database), newCentroids)) return undefined;
+    if (refWrites.length && !transactionMutate(refIndexStore(database), refWrites)) return undefined;
+    return true;
+}
+
+// Tombstone each ref in its cell (found via refIndex) and drop it from the index. undefined while not synced.
+export function removeEmbeddings(
+    database: Database<IvfEmbeddingRoot>,
+    refs: string[],
+): true | undefined {
+    if (!refs.length) return true;
+    const refIndex = transactionRead<string>(refIndexStore(database));
+    if (!refIndex) return undefined;
+
+    const refsByCell = new Map<string, string[]>();
+    for (const ref of refs) {
+        const cellId = refIndex.get(ref);
+        if (!cellId) continue;
+        let group = refsByCell.get(cellId);
+        if (!group) {
+            group = [];
+            refsByCell.set(cellId, group);
+        }
+        group.push(ref);
+    }
+
+    for (const [cellId, group] of refsByCell) {
+        const memberDeletes = group.map(ref => ({ key: ref, value: undefined }));
+        if (!transactionMutate<StoredEmbedding>(cellStore(database, cellId), memberDeletes)) return undefined;
+    }
+    const refDeletes = refs.map(ref => ({ key: ref, value: undefined }));
+    if (!transactionMutate<string>(refIndexStore(database), refDeletes)) return undefined;
+    return true;
+}

package/storage/proxydatabase/transactionSet.d.ts

@@ -3,6 +3,7 @@ export type TransactionSetStore = {
     [fileNumber: string]: Uint8Array;
 };
 export declare function transactionRead<Value>(database: Database<TransactionSetStore>): Map<string, Value> | undefined;
+export declare function replayTransactionStore<Value>(store: TransactionSetStore | undefined): Map<string, Value>;
 export declare function transactionMutate<Value>(database: Database<TransactionSetStore>, transactions: {
     key: string;
     value: Value | undefined;

package/storage/proxydatabase/transactionSet.ts

@@ -21,6 +21,17 @@ function maxNumber(values: number[], fallback: number): number {
     return result;
 }
 
+function decodeBuffers(buffers: Uint8Array[]): Transaction[] {
+    const transactions: Transaction[] = [];
+    for (const buffer of buffers) {
+        const decoded = transactionCbor.decode(buffer) as Transaction[];
+        for (const transaction of decoded) {
+            transactions.push(transaction);
+        }
+    }
+    return transactions;
+}
+
 function readTransactionFiles(
     database: Database<TransactionSetStore>,
 ): { fileKeys: string[]; transactions: Transaction[] } | undefined {
@@ -29,14 +40,7 @@ function readTransactionFiles(
         buffers: Object.values(store),
     }));
     if (!snapshot) return undefined;
-    const transactions: Transaction[] = [];
-    for (const buffer of snapshot.buffers) {
-        const decoded = transactionCbor.decode(buffer) as Transaction[];
-        for (const transaction of decoded) {
-            transactions.push(transaction);
-        }
-    }
-    return { fileKeys: snapshot.fileKeys, transactions };
+    return { fileKeys: snapshot.fileKeys, transactions: decodeBuffers(snapshot.buffers) };
 }
 
 function replay(transactions: Transaction[]): Map<string, unknown> {
@@ -77,6 +81,12 @@ export function transactionRead<Value>(
     return replay(files.transactions) as Map<string, Value>;
 }
 
+// Replay an already-read store object (no read of its own), so a caller can fetch many sets in one batched read and then materialize each.
+export function replayTransactionStore<Value>(store: TransactionSetStore | undefined): Map<string, Value> {
+    if (!store) return new Map();
+    return replay(decodeBuffers(Object.values(store))) as Map<string, Value>;
+}
+
 export function transactionMutate<Value>(
     database: Database<TransactionSetStore>,
     transactions: { key: string; value: Value | undefined }[],