sliftutils 1.4.81 → 1.4.83

package/storage/proxydatabase/ivfEmbeddingDatabase.ts

@@ -1,96 +1,219 @@
+import cborx from "cbor-x";
 import { Database, namespaceDatabase } from "./Database";
-import { TransactionSetStore, transactionRead, transactionMutate, replayTransactionStore } from "./transactionSet";
-import { StoredEmbedding, EmbeddingFormat, getCloseness, averageEmbeddings, hashEmbedding } from "../embeddingFormats";
+import { TransactionSetStore, transactionRead, transactionMutate, transactionDelete, replayTransactionStore } from "./transactionSet";
+import { StoredEmbedding, EmbeddingFormat, getCloseness, embeddingToFloat32, releaseFloat32, encodeEmbedding, hashEmbedding } from "../embeddingFormats";
 
 export type IvfConfig = {
     model: string;
     format: EmbeddingFormat;
-    cellTarget: number;
-    splitAt: number;
+    // Target number of embeddings per cell.
+    cellTargetSize: number;
 };
 
-// Cell ids ARE the hash of the cell's centroid, so there's no id counter. centroids maps that id to the
-// centroid embedding (read whole as the preload, ranked in RAM). Each cell is its own transaction set of
-// ref → member embedding. refIndex maps a ref to the cell holding it, so a remove can find it.
+// Tiered storage. Below FLAT_LIMIT embeddings the whole set lives in one flat transaction set (no IVF);
+// once it grows past that we build the IVF (centroids + per-cell sets) and stay there forever. `count` is
+// the live embedding count (re-derived exactly on a rebuild). `steps` records one-time upgrades that must
+// never re-run even if the set later shrinks and regrows.
 export type IvfEmbeddingRoot = {
     config: IvfConfig;
-    centroids: TransactionSetStore;
-    cells: { [cellId: string]: TransactionSetStore };
-    refIndex: TransactionSetStore;
+    count: number;
+    flat: TransactionSetStore<StoredEmbedding>;
+    // Direct ref -> embedding index. Each entry is one CBOR blob at its own key, so a lookup or delete is a
+    // single key-value op (no transaction-set replay) and deleteData can drop it (it's a primitive leaf). Refs
+    // and their embeddings never move, so a rebuild leaves this untouched.
+    byRef: { [ref: string]: Uint8Array };
+    steps: { [step: string]: boolean };
+    centroids: TransactionSetStore<StoredEmbedding>;
+    cells: { [cellId: string]: TransactionSetStore<StoredEmbedding> };
 };
 
 export type EmbeddingInput = { ref: string; embedding: StoredEmbedding };
 export type SearchHit = { ref: string; closeness: number };
+type CellEntry = { ref: string; embedding: StoredEmbedding };
 
-function centroidStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore> {
+const embeddingCbor = new cborx.Encoder({ structuredClone: true });
+function encodeRefValue(embedding: StoredEmbedding): Uint8Array {
+    return embeddingCbor.encode(embedding) as Uint8Array;
+}
+function decodeRefValue(value: Uint8Array): StoredEmbedding {
+    return embeddingCbor.decode(value) as StoredEmbedding;
+}
+
+// Below this many embeddings we skip the IVF entirely (flat store). 1024 = a clean power of two.
+const FLAT_LIMIT = 1024;
+// Force a from-scratch rebuild once at each of these (multiples of 4 above FLAT_LIMIT, up to ~16k), gated by a step flag so each runs at most once ever.
+const REGENERATE_AT = [4096, 16384];
+const STEP_IVF = "ivf";
+// On delete, the member's exact cell plus this many nearby cells are checked, in case a rebuild left it non-optimal.
+const DELETE_FALLBACK_CELLS = 10;
+const REBALANCE_ITERATIONS = 4;
+
+function flatStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore<StoredEmbedding>> {
+    return namespaceDatabase(database, root => root.flat);
+}
+function centroidStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore<StoredEmbedding>> {
     return namespaceDatabase(database, root => root.centroids);
 }
-function cellStore(database: Database<IvfEmbeddingRoot>, cellId: string): Database<TransactionSetStore> {
+function cellStore(database: Database<IvfEmbeddingRoot>, cellId: string): Database<TransactionSetStore<StoredEmbedding>> {
     return namespaceDatabase(database, root => root.cells[cellId]);
 }
-function refIndexStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore> {
-    return namespaceDatabase(database, root => root.refIndex);
+
+function rankCellsByCloseness(embedding: StoredEmbedding, centroids: Map<string, StoredEmbedding>): string[] {
+    const ranked: { cellId: string; closeness: number }[] = [];
+    for (const cellId of centroids.keys()) {
+        ranked.push({ cellId, closeness: getCloseness(embedding, centroids.get(cellId)!) });
+    }
+    ranked.sort((left, right) => right.closeness - left.closeness);
+    return ranked.map(entry => entry.cellId);
 }
 
-function nearestCell(embedding: StoredEmbedding, centroids: Map<string, StoredEmbedding>): string {
-    let bestCellId = "";
-    let bestCloseness = -Infinity;
-    for (const [cellId, centroid] of centroids) {
-        const closeness = getCloseness(embedding, centroid);
-        if (closeness > bestCloseness) {
-            bestCloseness = closeness;
-            bestCellId = cellId;
-        }
+// Per-write chance of a full rebuild. Zero at/under the target size, then rises (cubically) past it so cells
+// stay roughly between target and ~2x target.
+function rebalanceProbability(fillRatio: number): number {
+    if (fillRatio <= 1) {
+        return 0;
     }
-    return bestCellId;
+    const over = fillRatio - 1;
+    return Math.min(1, over * over * over * 0.25);
 }
 
-// 2-means on a cell's members so an overgrown cell can split in place. Seeds with one member and the one
-// farthest from it, then alternates assign / recompute-centroid a few times.
-function splitInTwo(
-    members: [string, StoredEmbedding][],
-    config: IvfConfig,
-): { centroid: StoredEmbedding; members: [string, StoredEmbedding][] }[] {
-    let centroidA = members[0][1];
-    let centroidB = members[0][1];
-    let worst = Infinity;
-    for (const [, embedding] of members) {
-        const closeness = getCloseness(centroidA, embedding);
-        if (closeness < worst) {
-            worst = closeness;
-            centroidB = embedding;
-        }
-    }
-    let groupA: [string, StoredEmbedding][] = [];
-    let groupB: [string, StoredEmbedding][] = [];
-    for (let iteration = 0; iteration < 5; iteration++) {
-        groupA = [];
-        groupB = [];
-        for (const member of members) {
-            if (getCloseness(centroidA, member[1]) >= getCloseness(centroidB, member[1])) {
-                groupA.push(member);
-            } else {
-                groupB.push(member);
+// k-means. Decodes every member to a pooled float32 buffer ONCE, then assigns with a plain internal float
+// dot (no getCloseness call — comparing two float vectors is trivial) and keeps centroids as float means,
+// encoding them to StoredEmbedding only at the end. Releases the borrowed buffers when done.
+function clusterMembers(members: CellEntry[], clusterCount: number, config: IvfConfig): { centroid: StoredEmbedding; members: CellEntry[] }[] {
+    const memberFloats: Float32Array[] = [];
+    for (const member of members) {
+        memberFloats.push(embeddingToFloat32(member.embedding, true));
+    }
+    const length = memberFloats.length ? memberFloats[0].length : 0;
+    let centroids: Float32Array[] = [];
+    const seedStep = members.length / clusterCount;
+    for (let clusterIndex = 0; clusterIndex < clusterCount; clusterIndex++) {
+        centroids.push(new Float32Array(memberFloats[Math.floor(clusterIndex * seedStep)]));
+    }
+    let groups: number[][] = [];
+    for (let iteration = 0; iteration < REBALANCE_ITERATIONS; iteration++) {
+        groups = [];
+        for (let clusterIndex = 0; clusterIndex < centroids.length; clusterIndex++) {
+            groups.push([]);
+        }
+        for (let memberIndex = 0; memberIndex < memberFloats.length; memberIndex++) {
+            const memberFloat = memberFloats[memberIndex];
+            let bestIndex = 0;
+            let bestDot = -Infinity;
+            for (let centroidIndex = 0; centroidIndex < centroids.length; centroidIndex++) {
+                const centroidFloat = centroids[centroidIndex];
+                let dot = 0;
+                for (let dim = 0; dim < length; dim++) {
+                    dot += memberFloat[dim] * centroidFloat[dim];
+                }
+                if (dot > bestDot) {
+                    bestDot = dot;
+                    bestIndex = centroidIndex;
+                }
             }
+            groups[bestIndex].push(memberIndex);
         }
-        if (!groupA.length || !groupB.length) {
-            break;
+        const nextCentroids: Float32Array[] = [];
+        const nextGroups: number[][] = [];
+        for (const group of groups) {
+            if (!group.length) {
+                continue;
+            }
+            const mean = new Float32Array(length);
+            for (const memberIndex of group) {
+                const memberFloat = memberFloats[memberIndex];
+                for (let dim = 0; dim < length; dim++) {
+                    mean[dim] += memberFloat[dim];
+                }
+            }
+            let norm = 0;
+            for (let dim = 0; dim < length; dim++) {
+                norm += mean[dim] * mean[dim];
+            }
+            const magnitude = Math.sqrt(norm) || 1;
+            for (let dim = 0; dim < length; dim++) {
+                mean[dim] /= magnitude;
+            }
+            nextCentroids.push(mean);
+            nextGroups.push(group);
         }
-        centroidA = averageEmbeddings(groupA.map(member => member[1]), config);
-        centroidB = averageEmbeddings(groupB.map(member => member[1]), config);
+        centroids = nextCentroids;
+        groups = nextGroups;
     }
-    const result: { centroid: StoredEmbedding; members: [string, StoredEmbedding][] }[] = [];
-    if (groupA.length) {
-        result.push({ centroid: centroidA, members: groupA });
+    const result: { centroid: StoredEmbedding; members: CellEntry[] }[] = [];
+    for (let clusterIndex = 0; clusterIndex < centroids.length; clusterIndex++) {
+        const centroid = encodeEmbedding({ input: centroids[clusterIndex], format: config.format, model: config.model });
+        const cellMembers: CellEntry[] = [];
+        for (const memberIndex of groups[clusterIndex]) {
+            cellMembers.push(members[memberIndex]);
+        }
+        result.push({ centroid, members: cellMembers });
     }
-    if (groupB.length) {
-        result.push({ centroid: centroidB, members: groupB });
+    for (const memberFloat of memberFloats) {
+        releaseFloat32(memberFloat);
     }
     return result;
 }
 
-// Nearest cells until probeBudget members are covered, scored exactly against the query. undefined while
-// not synced. All probed cells are fetched in one batched read so probing doesn't cascade per cell.
+// Re-cluster EVERY embedding (flat tier + all cells) into a fresh IVF, clear the flat tier, and mark IVF
+// mode. This is the rearranger: run probabilistically as the set grows, once at each regenerate threshold,
+// and once to upgrade from flat. Reads everything it uses, so it bails (does nothing) if anything isn't
+// synced and the caller's retry re-runs it; no caller cares about its result.
+export function rebuildStructure(database: Database<IvfEmbeddingRoot>): void {
+    const config = database.readData(root => root.config);
+    if (!config) return;
+    const steps = database.readData(root => root.steps);
+    if (!steps) return;
+
+    const allMembers: CellEntry[] = [];
+    // Only the one-time flat->IVF upgrade pulls from the flat tier. Once STEP_IVF is set the flat data has
+    // already been folded into cells, so later rebuilds ignore it. It's left in place, never deleted: deleteData
+    // only removes primitive leaves, not a whole store, and the flat tier is small enough not to matter.
+    if (!steps[STEP_IVF]) {
+        const flat = transactionRead(flatStore(database));
+        if (!flat) return;
+        for (const ref of flat.keys()) {
+            allMembers.push({ ref, embedding: flat.get(ref)! });
+        }
+    }
+    const centroids = transactionRead(centroidStore(database));
+    if (!centroids) return;
+    const oldCellIds = Array.from(centroids.keys());
+    const cellStores = database.readData(root => oldCellIds.map(cellId => root.cells[cellId]));
+    if (!cellStores) return;
+    for (const store of cellStores) {
+        const members = replayTransactionStore<StoredEmbedding>(store);
+        for (const ref of members.keys()) {
+            allMembers.push({ ref, embedding: members.get(ref)! });
+        }
+    }
+    if (!allMembers.length) return;
+
+    const clusterCount = Math.max(1, Math.round(allMembers.length / config.cellTargetSize));
+    const clusters = clusterMembers(allMembers, clusterCount, config);
+
+    const newCellIds = new Set<string>();
+    const centroidWrites: { key: string; value: StoredEmbedding | undefined }[] = [];
+    for (const cluster of clusters) {
+        const cellId = hashEmbedding(cluster.centroid);
+        newCellIds.add(cellId);
+        centroidWrites.push({ key: cellId, value: cluster.centroid });
+        const memberWrites = cluster.members.map(member => ({ key: member.ref, value: member.embedding }));
+        transactionMutate(cellStore(database, cellId), memberWrites);
+    }
+    for (const oldCellId of oldCellIds) {
+        if (newCellIds.has(oldCellId)) {
+            continue;
+        }
+        centroidWrites.push({ key: oldCellId, value: undefined });
+        transactionDelete(cellStore(database, oldCellId));
+    }
+    transactionMutate(centroidStore(database), centroidWrites);
+    database.writeData(root => root.count, allMembers.length);
+    database.writeData(root => root.steps[STEP_IVF], true);
+}
+
 export function searchEmbeddings(
     database: Database<IvfEmbeddingRoot>,
     query: StoredEmbedding,
@@ -98,54 +221,86 @@ export function searchEmbeddings(
 ): SearchHit[] | undefined {
     const config = database.readData(root => root.config);
     if (!config) return undefined;
-    const centroids = transactionRead<StoredEmbedding>(centroidStore(database));
-    if (!centroids) return undefined;
-    if (!centroids.size) return [];
+    const steps = database.readData(root => root.steps);
+    if (!steps) return undefined;
 
-    const rankedCellIds = [...centroids.entries()]
-        .map(([cellId, centroid]) => ({ cellId, closeness: getCloseness(query, centroid) }))
-        .sort((left, right) => right.closeness - left.closeness)
-        .map(ranked => ranked.cellId);
+    const hits: SearchHit[] = [];
+    if (!steps[STEP_IVF]) {
+        const flat = transactionRead(flatStore(database));
+        if (!flat) return undefined;
+        for (const ref of flat.keys()) {
+            hits.push({ ref, closeness: getCloseness(query, flat.get(ref)!) });
+        }
+        hits.sort((left, right) => right.closeness - left.closeness);
+        return hits.slice(0, options.resultCount);
+    }
 
-    const probeCellCount = Math.max(1, Math.ceil(options.probeBudget / config.cellTarget));
-    const probeCellIds = rankedCellIds.slice(0, probeCellCount);
+    const centroids = transactionRead(centroidStore(database));
+    if (!centroids) return undefined;
+    if (!centroids.size) return [];
+    const probeCellCount = Math.max(1, Math.ceil(options.probeBudget / config.cellTargetSize));
+    const probeCellIds = rankCellsByCloseness(query, centroids).slice(0, probeCellCount);
     const stores = database.readData(root => probeCellIds.map(cellId => root.cells[cellId]));
     if (!stores) return undefined;
-
-    const hits: SearchHit[] = [];
     for (const store of stores) {
         const members = replayTransactionStore<StoredEmbedding>(store);
-        for (const [ref, embedding] of members) {
-            hits.push({ ref, closeness: getCloseness(query, embedding) });
+        for (const ref of members.keys()) {
+            hits.push({ ref, closeness: getCloseness(query, members.get(ref)!) });
         }
     }
     hits.sort((left, right) => right.closeness - left.closeness);
     return hits.slice(0, options.resultCount);
 }
 
-// Assign each embedding to its nearest cell and add it. An empty index bootstraps a first cell from the
-// inserted batch; a cell that crosses splitAt*cellTarget splits in place. undefined while not synced.
+// Direct ref -> stored embedding lookup through the byRef index: one key-value read per ref, no cell scan. A
+// search returns refs; this fetches the data behind them. Skips refs that aren't present; undefined if not synced.
+export function lookupEmbeddings(
+    database: Database<IvfEmbeddingRoot>,
+    refs: string[],
+): Map<string, StoredEmbedding> | undefined {
+    const values = database.readData(root => refs.map(ref => root.byRef[ref]));
+    if (!values) return undefined;
+    const result = new Map<string, StoredEmbedding>();
+    for (let index = 0; index < refs.length; index++) {
+        const value = values[index];
+        if (value) {
+            result.set(refs[index], decodeRefValue(value));
+        }
+    }
+    return result;
+}
+
 export function insertEmbeddings(
     database: Database<IvfEmbeddingRoot>,
     items: EmbeddingInput[],
-): true | undefined {
-    if (!items.length) return true;
+) {
+    if (!items.length) return;
     const config = database.readData(root => root.config);
-    if (!config) return undefined;
-    const centroids = transactionRead<StoredEmbedding>(centroidStore(database));
-    if (!centroids) return undefined;
+    const steps = database.readData(root => root.steps);
+    let count = database.readData(root => root.count);
+    if (!config) return;
+    if (!steps) return;
+    if (count === undefined) return;
+    const newCount = count + items.length;
 
-    const newCentroids: { key: string; value: StoredEmbedding | undefined }[] = [];
-    if (!centroids.size) {
-        const centroid = averageEmbeddings(items.map(item => item.embedding), config);
-        const cellId = hashEmbedding(centroid);
-        centroids.set(cellId, centroid);
-        newCentroids.push({ key: cellId, value: centroid });
+    if (!steps[STEP_IVF]) {
+        const flatWrites = items.map(item => ({ key: item.ref, value: item.embedding }));
+        transactionMutate(flatStore(database), flatWrites);
+        for (const item of items) {
+            database.writeData(root => root.byRef[item.ref], encodeRefValue(item.embedding));
+        }
+        database.writeData(root => root.count, newCount);
+        if (newCount > FLAT_LIMIT) {
+            rebuildStructure(database);
+        }
+        return;
     }
 
+    const centroids = transactionRead(centroidStore(database));
+    if (!centroids) return undefined;
     const itemsByCell = new Map<string, EmbeddingInput[]>();
     for (const item of items) {
-        const cellId = nearestCell(item.embedding, centroids);
+        const cellId = rankCellsByCloseness(item.embedding, centroids)[0];
         let group = itemsByCell.get(cellId);
         if (!group) {
             group = [];
@@ -153,75 +308,117 @@ export function insertEmbeddings(
         }
         group.push(item);
     }
-
-    const targetCellIds = [...itemsByCell.keys()];
-    const existingStores = database.readData(root => targetCellIds.map(cellId => root.cells[cellId]));
-    if (!existingStores) return undefined;
-
-    const splitThreshold = config.splitAt * config.cellTarget;
-    const refWrites: { key: string; value: string | undefined }[] = [];
-    for (let cellIndex = 0; cellIndex < targetCellIds.length; cellIndex++) {
-        const cellId = targetCellIds[cellIndex];
+    for (const cellId of itemsByCell.keys()) {
         const group = itemsByCell.get(cellId)!;
-        const combined = replayTransactionStore<StoredEmbedding>(existingStores[cellIndex]);
-        for (const item of group) {
-            combined.set(item.ref, item.embedding);
-        }
+        const memberWrites = group.map(item => ({ key: item.ref, value: item.embedding }));
+        transactionMutate(cellStore(database, cellId), memberWrites);
+    }
+    for (const item of items) {
+        database.writeData(root => root.byRef[item.ref], encodeRefValue(item.embedding));
+    }
+    database.writeData(root => root.count, newCount);
 
-        if (combined.size <= splitThreshold) {
-            const memberWrites = group.map(item => ({ key: item.ref, value: item.embedding }));
-            if (!transactionMutate(cellStore(database, cellId), memberWrites)) return undefined;
-            for (const item of group) {
-                refWrites.push({ key: item.ref, value: cellId });
-            }
-            continue;
-        }
 
-        const subCells = splitInTwo([...combined.entries()], config);
-        for (const subCell of subCells) {
-            const subCellId = hashEmbedding(subCell.centroid);
-            newCentroids.push({ key: subCellId, value: subCell.centroid });
-            const memberWrites = subCell.members.map(([ref, embedding]) => ({ key: ref, value: embedding }));
-            if (!transactionMutate(cellStore(database, subCellId), memberWrites)) return undefined;
-            for (const [ref] of subCell.members) {
-                refWrites.push({ key: ref, value: subCellId });
+    const shouldRegenerate = (database: Database<IvfEmbeddingRoot>) => {
+        let steps = database.readData(root => root.steps);
+        if (!steps) return;
+        for (const threshold of REGENERATE_AT) {
+            const stepName = "regen" + threshold;
+            if (!steps[stepName] && newCount > threshold) {
+                database.writeData(root => root.steps[stepName], true);
+                return true;
             }
         }
-        newCentroids.push({ key: cellId, value: undefined });
-        database.deleteData(root => root.cells[cellId]);
+        const averageFill = newCount / Math.max(1, centroids.size) / config.cellTargetSize;
+        if (Math.random() < rebalanceProbability(averageFill)) {
+            rebuildStructure(database);
+        }
+        return true;
+    };
+    if (shouldRegenerate(database)) {
+        rebuildStructure(database);
     }
-
-    if (newCentroids.length && !transactionMutate(centroidStore(database), newCentroids)) return undefined;
-    if (refWrites.length && !transactionMutate(refIndexStore(database), refWrites)) return undefined;
-    return true;
 }
 
-// Tombstone each ref in its cell (found via refIndex) and drop it from the index. undefined while not synced.
 export function removeEmbeddings(
     database: Database<IvfEmbeddingRoot>,
     refs: string[],
-): true | undefined {
-    if (!refs.length) return true;
-    const refIndex = transactionRead<string>(refIndexStore(database));
-    if (!refIndex) return undefined;
-
-    const refsByCell = new Map<string, string[]>();
-    for (const ref of refs) {
-        const cellId = refIndex.get(ref);
-        if (!cellId) continue;
-        let group = refsByCell.get(cellId);
-        if (!group) {
-            group = [];
-            refsByCell.set(cellId, group);
+) {
+    if (!refs.length) return;
+    const steps = database.readData(root => root.steps);
+    const count = database.readData(root => root.count);
+    if (!steps) return;
+    if (count === undefined) return;
+
+    if (!steps[STEP_IVF]) {
+        const flatDeletes = refs.map(ref => ({ key: ref, value: undefined }));
+        transactionMutate(flatStore(database), flatDeletes);
+        for (const ref of refs) {
+            database.deleteData(root => root.byRef[ref]);
+        }
+        database.writeData(root => root.count, Math.max(0, count - refs.length));
+        return;
+    }
+
+    const centroids = transactionRead(centroidStore(database));
+    if (!centroids) return;
+    if (!centroids.size) return;
+
+    // Look up each ref's embedding through byRef, then rank its cells the way an insert would, so we delete from
+    // the cell it actually landed in (plus a few neighbours, in case a rebuild nudged it across a boundary).
+    const refValues = database.readData(root => refs.map(ref => root.byRef[ref]));
+    if (!refValues) return;
+    const candidatesByItem: { ref: string; cellIds: string[] }[] = [];
+    const candidateSet = new Set<string>();
+    for (let index = 0; index < refs.length; index++) {
+        const value = refValues[index];
+        if (!value) {
+            continue;
+        }
+        const embedding = decodeRefValue(value);
+        const cellIds = rankCellsByCloseness(embedding, centroids).slice(0, 1 + DELETE_FALLBACK_CELLS);
+        candidatesByItem.push({ ref: refs[index], cellIds });
+        for (const cellId of cellIds) {
+            candidateSet.add(cellId);
         }
-        group.push(ref);
     }
 
-    for (const [cellId, group] of refsByCell) {
-        const memberDeletes = group.map(ref => ({ key: ref, value: undefined }));
-        if (!transactionMutate<StoredEmbedding>(cellStore(database, cellId), memberDeletes)) return undefined;
+    const candidateCellIds = Array.from(candidateSet);
+    const stores = database.readData(root => candidateCellIds.map(cellId => root.cells[cellId]));
+    if (!stores) return;
+    const membersByCell = new Map<string, Map<string, StoredEmbedding>>();
+    for (let index = 0; index < candidateCellIds.length; index++) {
+        membersByCell.set(candidateCellIds[index], replayTransactionStore(stores[index]));
+    }
+
+    const deletesByCell = new Map<string, string[]>();
+    const deletedRefs: string[] = [];
+    for (const candidate of candidatesByItem) {
+        for (const cellId of candidate.cellIds) {
+            const members = membersByCell.get(cellId);
+            if (!members || !members.has(candidate.ref)) {
+                continue;
+            }
+            let cellRefs = deletesByCell.get(cellId);
+            if (!cellRefs) {
+                cellRefs = [];
+                deletesByCell.set(cellId, cellRefs);
+            }
+            cellRefs.push(candidate.ref);
+            deletedRefs.push(candidate.ref);
+            break;
+        }
+    }
+
+    for (const cellId of deletesByCell.keys()) {
+        const cellRefs = deletesByCell.get(cellId)!;
+        const memberDeletes = cellRefs.map(ref => ({ key: ref, value: undefined }));
+        transactionMutate(cellStore(database, cellId), memberDeletes);
+    }
+    for (const ref of deletedRefs) {
+        database.deleteData(root => root.byRef[ref]);
+    }
+    if (deletedRefs.length) {
+        database.writeData(root => root.count, Math.max(0, count - deletedRefs.length));
     }
-    const refDeletes = refs.map(ref => ({ key: ref, value: undefined }));
-    if (!transactionMutate<string>(refIndexStore(database), refDeletes)) return undefined;
-    return true;
 }

package/storage/proxydatabase/transactionSet.d.ts

@@ -1,10 +1,14 @@
 import { Database } from "./Database";
-export type TransactionSetStore = {
+declare const valueTag: unique symbol;
+export type TransactionSetStore<Value> = {
     [fileNumber: string]: Uint8Array;
+    [valueTag]?: Value;
 };
-export declare function transactionRead<Value>(database: Database<TransactionSetStore>): Map<string, Value> | undefined;
-export declare function replayTransactionStore<Value>(store: TransactionSetStore | undefined): Map<string, Value>;
-export declare function transactionMutate<Value>(database: Database<TransactionSetStore>, transactions: {
+export declare function transactionRead<Value>(database: Database<TransactionSetStore<Value>>): Map<string, Value> | undefined;
+export declare function replayTransactionStore<Value>(store: TransactionSetStore<Value> | undefined): Map<string, Value>;
+export declare function transactionMutate<Value>(database: Database<TransactionSetStore<Value>>, transactions: {
     key: string;
     value: Value | undefined;
-}[], compactAfterFiles?: number): true | undefined;
+}[], compactAfterFiles?: number): void;
+export declare function transactionDelete<Value>(database: Database<TransactionSetStore<Value>>): void;
+export {};