sliftutils 1.4.80 → 1.4.82

package/storage/proxydatabase/inMemoryDatabase.ts

@@ -0,0 +1,108 @@
+import { Database } from "./Database";
+
+// Rough byte size of a value, for the read/written-bytes metrics.
+function sizeOf(value: unknown): number {
+    if (value === undefined || value === null) {
+        return 0;
+    }
+    if (value instanceof Uint8Array) {
+        return value.byteLength;
+    }
+    if (typeof value === "string") {
+        return value.length;
+    }
+    if (typeof value === "number") {
+        return 8;
+    }
+    if (typeof value === "boolean") {
+        return 1;
+    }
+    if (Array.isArray(value)) {
+        let total = 0;
+        for (const entry of value) {
+            total += sizeOf(entry);
+        }
+        return total;
+    }
+    if (typeof value === "object") {
+        let total = 0;
+        const record = value as Record<string, unknown>;
+        for (const key of Object.keys(record)) {
+            total += sizeOf(record[key]);
+        }
+        return total;
+    }
+    return 0;
+}
+
+// Record the property path a dereference lambda walks, so a write/delete can be applied at it.
+function capturePath(deref: (root: unknown) => unknown): string[] {
+    const path: string[] = [];
+    const handler: ProxyHandler<object> = {
+        get(_target, key) {
+            if (typeof key === "string") {
+                path.push(key);
+            }
+            return new Proxy({}, handler);
+        },
+    };
+    deref(new Proxy({}, handler));
+    return path;
+}
+
+function setPath(root: Record<string, unknown>, path: string[], value: unknown): void {
+    let node = root;
+    for (let index = 0; index < path.length - 1; index++) {
+        const key = path[index];
+        const next = node[key];
+        if (!next || typeof next !== "object") {
+            node[key] = {};
+        }
+        node = node[key] as Record<string, unknown>;
+    }
+    node[path[path.length - 1]] = value;
+}
+
+function deletePath(root: Record<string, unknown>, path: string[]): void {
+    let node = root;
+    for (let index = 0; index < path.length - 1; index++) {
+        const next = node[path[index]];
+        if (!next || typeof next !== "object") {
+            return;
+        }
+        node = next as Record<string, unknown>;
+    }
+    delete node[path[path.length - 1]];
+}
+
+// In-memory Database for tests/dev: holds the whole root as a plain object and counts every call plus the
+// bytes flowing through. Synchronous and always "synced", so reads only return undefined for a genuinely
+// missing scalar (a missing transaction set reads as empty inside the set helpers).
+export class InMemoryDatabase<Root> implements Database<Root> {
+    public readCalls = 0;
+    public writeCalls = 0;
+    public deleteCalls = 0;
+    public bytesRead = 0;
+    public bytesWritten = 0;
+    private root: Record<string, unknown>;
+
+    constructor(initial: Root) {
+        this.root = initial as Record<string, unknown>;
+    }
+
+    readData<Value>(deref: (root: Root) => Value): Value | undefined {
+        this.readCalls++;
+        const result = deref(this.root as Root);
+        this.bytesRead += sizeOf(result);
+        return result;
+    }
+    writeData<Value>(deref: (root: Root) => Value, newValue: Value): void {
+        this.writeCalls++;
+        this.bytesWritten += sizeOf(newValue);
+        setPath(this.root, capturePath(deref as (root: unknown) => unknown), newValue);
+    }
+    deleteData(deref: (root: Root) => unknown): void {
+        this.deleteCalls++;
+        deletePath(this.root, capturePath(deref as (root: unknown) => unknown));
+    }
+}

package/storage/proxydatabase/ivfDbCheck.d.ts

@@ -0,0 +1 @@
+export {};

package/storage/proxydatabase/ivfDbCheck.ts

@@ -0,0 +1,85 @@
+import fs from "fs";
+import { InMemoryDatabase } from "./inMemoryDatabase";
+import { IvfEmbeddingRoot, IvfConfig, EmbeddingInput, insertEmbeddings, searchEmbeddings } from "./ivfEmbeddingDatabase";
+import { encodeEmbedding, EmbeddingFormat } from "../embeddingFormats";
+import { ensureFaceData } from "../faceFramesData";
+
+// Exercises the tiered IVF embedding database through the in-memory Database wrapper, which counts every
+// read/write/delete call and the bytes through each, across dataset sizes and load patterns, with timing.
+
+const DIM = 512;
+const FILE_NAME = "faceFrames2_full.f32";
+const MODEL = "buffalo_l";
+const FORMAT: EmbeddingFormat = "q8g8_2048";
+
+async function loadFloats(count: number): Promise<Float32Array> {
+    const byteLength = count * DIM * 4;
+    const filePath = await ensureFaceData(FILE_NAME, byteLength);
+    const buffer = Buffer.alloc(byteLength);
+    const handle = fs.openSync(filePath, "r");
+    fs.readSync(handle, buffer, 0, byteLength, 0);
+    fs.closeSync(handle);
+    return new Float32Array(buffer.buffer, buffer.byteOffset, count * DIM);
+}
+
+async function buildInputs(count: number): Promise<EmbeddingInput[]> {
+    const floats = await loadFloats(count);
+    const inputs: EmbeddingInput[] = [];
+    for (let index = 0; index < count; index++) {
+        const slice = floats.subarray(index * DIM, (index + 1) * DIM);
+        inputs.push({ ref: "e" + index, embedding: encodeEmbedding({ input: slice, format: FORMAT, model: MODEL }) });
+    }
+    return inputs;
+}
+
+function freshDatabase(config: IvfConfig): InMemoryDatabase<IvfEmbeddingRoot> {
+    const root: IvfEmbeddingRoot = { config, count: 0, flat: {}, steps: {}, centroids: {}, cells: {} };
+    return new InMemoryDatabase<IvfEmbeddingRoot>(root);
+}
+
+function pad(value: string | number, width: number): string {
+    return String(value).padStart(width);
+}
+
+async function main() {
+    const config: IvfConfig = { model: MODEL, format: FORMAT, cellTargetSize: 128 };
+    const runs: { size: number; batch: number }[] = [
+        { size: 10000, batch: 1 },
+        { size: 10000, batch: 100 },
+        { size: 100000, batch: 1000 },
+    ];
+
+    let maxSize = 0;
+    for (const run of runs) {
+        if (run.size > maxSize) {
+            maxSize = run.size;
+        }
+    }
+    const loadStart = Date.now();
+    const allInputs = await buildInputs(maxSize);
+    console.log(`loaded + encoded ${maxSize} embeddings in ${((Date.now() - loadStart) / 1000).toFixed(1)}s\n`);
+
+    console.log(`${pad("added", 7)} ${pad("batch", 5)} | ${pad("secs", 7)} ${pad("reads", 8)} ${pad("writes", 8)} ${pad("deletes", 8)} ${pad("readMB", 9)} ${pad("writeMB", 8)} | recall`);
+    for (const run of runs) {
+        const database = freshDatabase(config);
+        const start = Date.now();
+        for (let offset = 0; offset < run.size; offset += run.batch) {
+            insertEmbeddings(database, allInputs.slice(offset, offset + run.batch));
+        }
+        const seconds = (Date.now() - start) / 1000;
+
+        let found = 0;
+        const sampleCount = 100;
+        for (let index = 0; index < sampleCount; index++) {
+            const query = allInputs[Math.floor(index * run.size / sampleCount)];
+            const hits = searchEmbeddings(database, query.embedding, { probeBudget: 512, resultCount: 1 });
+            if (hits && hits.length && hits[0].ref === query.ref) {
+                found++;
+            }
+        }
+        const recall = (found / sampleCount).toFixed(2);
+        console.log(`${pad(run.size, 7)} ${pad(run.batch, 5)} | ${pad(seconds.toFixed(1), 7)} ${pad(database.readCalls, 8)} ${pad(database.writeCalls, 8)} ${pad(database.deleteCalls, 8)} ${pad((database.bytesRead / 1e6).toFixed(0), 9)} ${pad((database.bytesWritten / 1e6).toFixed(0), 8)} | ${recall}`);
+    }
+}
+
+void main();

package/storage/proxydatabase/ivfEmbeddingDatabase.d.ts

@@ -0,0 +1,35 @@
+import { Database } from "./Database";
+import { TransactionSetStore } from "./transactionSet";
+import { StoredEmbedding, EmbeddingFormat } from "../embeddingFormats";
+export type IvfConfig = {
+    model: string;
+    format: EmbeddingFormat;
+    cellTargetSize: number;
+};
+export type IvfEmbeddingRoot = {
+    config: IvfConfig;
+    count: number;
+    flat: TransactionSetStore<StoredEmbedding>;
+    steps: {
+        [step: string]: boolean;
+    };
+    centroids: TransactionSetStore<StoredEmbedding>;
+    cells: {
+        [cellId: string]: TransactionSetStore<StoredEmbedding>;
+    };
+};
+export type EmbeddingInput = {
+    ref: string;
+    embedding: StoredEmbedding;
+};
+export type SearchHit = {
+    ref: string;
+    closeness: number;
+};
+export declare function rebuildStructure(database: Database<IvfEmbeddingRoot>): void;
+export declare function searchEmbeddings(database: Database<IvfEmbeddingRoot>, query: StoredEmbedding, options: {
+    probeBudget: number;
+    resultCount: number;
+}): SearchHit[] | undefined;
+export declare function insertEmbeddings(database: Database<IvfEmbeddingRoot>, items: EmbeddingInput[]): undefined;
+export declare function removeEmbeddings(database: Database<IvfEmbeddingRoot>, items: EmbeddingInput[]): void;

package/storage/proxydatabase/ivfEmbeddingDatabase.ts

@@ -0,0 +1,372 @@
+import { Database, namespaceDatabase } from "./Database";
+import { TransactionSetStore, transactionRead, transactionMutate, transactionDelete, replayTransactionStore } from "./transactionSet";
+import { StoredEmbedding, EmbeddingFormat, getCloseness, embeddingToFloat32, releaseFloat32, encodeEmbedding, hashEmbedding } from "../embeddingFormats";
+
+export type IvfConfig = {
+    model: string;
+    format: EmbeddingFormat;
+    // Target number of embeddings per cell.
+    cellTargetSize: number;
+};
+
+// Tiered storage. Below FLAT_LIMIT embeddings the whole set lives in one flat transaction set (no IVF);
+// once it grows past that we build the IVF (centroids + per-cell sets) and stay there forever. `count` is
+// the live embedding count (re-derived exactly on a rebuild). `steps` records one-time upgrades that must
+// never re-run even if the set later shrinks and regrows.
+export type IvfEmbeddingRoot = {
+    config: IvfConfig;
+    count: number;
+    flat: TransactionSetStore<StoredEmbedding>;
+    steps: { [step: string]: boolean };
+    centroids: TransactionSetStore<StoredEmbedding>;
+    cells: { [cellId: string]: TransactionSetStore<StoredEmbedding> };
+};
+
+export type EmbeddingInput = { ref: string; embedding: StoredEmbedding };
+export type SearchHit = { ref: string; closeness: number };
+type CellEntry = { ref: string; embedding: StoredEmbedding };
+
+// Below this many embeddings we skip the IVF entirely (flat store). 1024 = a clean power of two.
+const FLAT_LIMIT = 1024;
+// Force a from-scratch rebuild once at each of these (multiples of 4 above FLAT_LIMIT, up to ~16k), gated by a step flag so each runs at most once ever.
+const REGENERATE_AT = [4096, 16384];
+const STEP_IVF = "ivf";
+// On delete, the member's exact cell plus this many nearby cells are checked, in case a rebuild left it non-optimal.
+const DELETE_FALLBACK_CELLS = 10;
+const REBALANCE_ITERATIONS = 4;
+
+function flatStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore<StoredEmbedding>> {
+    return namespaceDatabase(database, root => root.flat);
+}
+function centroidStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore<StoredEmbedding>> {
+    return namespaceDatabase(database, root => root.centroids);
+}
+function cellStore(database: Database<IvfEmbeddingRoot>, cellId: string): Database<TransactionSetStore<StoredEmbedding>> {
+    return namespaceDatabase(database, root => root.cells[cellId]);
+}
+
+function rankCellsByCloseness(embedding: StoredEmbedding, centroids: Map<string, StoredEmbedding>): string[] {
+    const ranked: { cellId: string; closeness: number }[] = [];
+    for (const cellId of centroids.keys()) {
+        ranked.push({ cellId, closeness: getCloseness(embedding, centroids.get(cellId)!) });
+    }
+    ranked.sort((left, right) => right.closeness - left.closeness);
+    return ranked.map(entry => entry.cellId);
+}
+
+// Per-write chance of a full rebuild. Zero at/under the target size, then rises (cubically) past it so cells
+// stay roughly between target and ~2x target.
+function rebalanceProbability(fillRatio: number): number {
+    if (fillRatio <= 1) {
+        return 0;
+    }
+    const over = fillRatio - 1;
+    return Math.min(1, over * over * over * 0.25);
+}
+
+// k-means. Decodes every member to a pooled float32 buffer ONCE, then assigns with a plain internal float
+// dot (no getCloseness call — comparing two float vectors is trivial) and keeps centroids as float means,
+// encoding them to StoredEmbedding only at the end. Releases the borrowed buffers when done.
+function clusterMembers(members: CellEntry[], clusterCount: number, config: IvfConfig): { centroid: StoredEmbedding; members: CellEntry[] }[] {
+    const memberFloats: Float32Array[] = [];
+    for (const member of members) {
+        memberFloats.push(embeddingToFloat32(member.embedding, true));
+    }
+    const length = memberFloats.length ? memberFloats[0].length : 0;
+    let centroids: Float32Array[] = [];
+    const seedStep = members.length / clusterCount;
+    for (let clusterIndex = 0; clusterIndex < clusterCount; clusterIndex++) {
+        centroids.push(new Float32Array(memberFloats[Math.floor(clusterIndex * seedStep)]));
+    }
+    let groups: number[][] = [];
+    for (let iteration = 0; iteration < REBALANCE_ITERATIONS; iteration++) {
+        groups = [];
+        for (let clusterIndex = 0; clusterIndex < centroids.length; clusterIndex++) {
+            groups.push([]);
+        }
+        for (let memberIndex = 0; memberIndex < memberFloats.length; memberIndex++) {
+            const memberFloat = memberFloats[memberIndex];
+            let bestIndex = 0;
+            let bestDot = -Infinity;
+            for (let centroidIndex = 0; centroidIndex < centroids.length; centroidIndex++) {
+                const centroidFloat = centroids[centroidIndex];
+                let dot = 0;
+                for (let dim = 0; dim < length; dim++) {
+                    dot += memberFloat[dim] * centroidFloat[dim];
+                }
+                if (dot > bestDot) {
+                    bestDot = dot;
+                    bestIndex = centroidIndex;
+                }
+            }
+            groups[bestIndex].push(memberIndex);
+        }
+        const nextCentroids: Float32Array[] = [];
+        const nextGroups: number[][] = [];
+        for (const group of groups) {
+            if (!group.length) {
+                continue;
+            }
+            const mean = new Float32Array(length);
+            for (const memberIndex of group) {
+                const memberFloat = memberFloats[memberIndex];
+                for (let dim = 0; dim < length; dim++) {
+                    mean[dim] += memberFloat[dim];
+                }
+            }
+            let norm = 0;
+            for (let dim = 0; dim < length; dim++) {
+                norm += mean[dim] * mean[dim];
+            }
+            const magnitude = Math.sqrt(norm) || 1;
+            for (let dim = 0; dim < length; dim++) {
+                mean[dim] /= magnitude;
+            }
+            nextCentroids.push(mean);
+            nextGroups.push(group);
+        }
+        centroids = nextCentroids;
+        groups = nextGroups;
+    }
+    const result: { centroid: StoredEmbedding; members: CellEntry[] }[] = [];
+    for (let clusterIndex = 0; clusterIndex < centroids.length; clusterIndex++) {
+        const centroid = encodeEmbedding({ input: centroids[clusterIndex], format: config.format, model: config.model });
+        const cellMembers: CellEntry[] = [];
+        for (const memberIndex of groups[clusterIndex]) {
+            cellMembers.push(members[memberIndex]);
+        }
+        result.push({ centroid, members: cellMembers });
+    }
+    for (const memberFloat of memberFloats) {
+        releaseFloat32(memberFloat);
+    }
+    return result;
+}
+
+// Re-cluster EVERY embedding (flat tier + all cells) into a fresh IVF, clear the flat tier, and mark IVF
+// mode. This is the rearranger: run probabilistically as the set grows, once at each regenerate threshold,
+// and once to upgrade from flat. Reads everything it uses, so it bails (does nothing) if anything isn't
+// synced and the caller's retry re-runs it; no caller cares about its result.
+export function rebuildStructure(database: Database<IvfEmbeddingRoot>): void {
+    const config = database.readData(root => root.config);
+    if (!config) return;
+    const steps = database.readData(root => root.steps);
+    if (!steps) return;
+
+    const allMembers: CellEntry[] = [];
+    // Only the one-time flat->IVF upgrade pulls from the flat tier. Once STEP_IVF is set the flat data has
+    // already been folded into cells, so later rebuilds ignore it. It's left in place, never deleted: deleteData
+    // only removes primitive leaves, not a whole store, and the flat tier is small enough not to matter.
+    if (!steps[STEP_IVF]) {
+        const flat = transactionRead(flatStore(database));
+        if (!flat) return;
+        for (const ref of flat.keys()) {
+            allMembers.push({ ref, embedding: flat.get(ref)! });
+        }
+    }
+    const centroids = transactionRead(centroidStore(database));
+    if (!centroids) return;
+    const oldCellIds = Array.from(centroids.keys());
+    const cellStores = database.readData(root => oldCellIds.map(cellId => root.cells[cellId]));
+    if (!cellStores) return;
+    for (const store of cellStores) {
+        const members = replayTransactionStore<StoredEmbedding>(store);
+        for (const ref of members.keys()) {
+            allMembers.push({ ref, embedding: members.get(ref)! });
+        }
+    }
+    if (!allMembers.length) return;
+
+    const clusterCount = Math.max(1, Math.round(allMembers.length / config.cellTargetSize));
+    const clusters = clusterMembers(allMembers, clusterCount, config);
+
+    const newCellIds = new Set<string>();
+    const centroidWrites: { key: string; value: StoredEmbedding | undefined }[] = [];
+    for (const cluster of clusters) {
+        const cellId = hashEmbedding(cluster.centroid);
+        newCellIds.add(cellId);
+        centroidWrites.push({ key: cellId, value: cluster.centroid });
+        const memberWrites = cluster.members.map(member => ({ key: member.ref, value: member.embedding }));
+        transactionMutate(cellStore(database, cellId), memberWrites);
+    }
+    for (const oldCellId of oldCellIds) {
+        if (newCellIds.has(oldCellId)) {
+            continue;
+        }
+        centroidWrites.push({ key: oldCellId, value: undefined });
+        transactionDelete(cellStore(database, oldCellId));
+    }
+    transactionMutate(centroidStore(database), centroidWrites);
+    database.writeData(root => root.count, allMembers.length);
+    database.writeData(root => root.steps[STEP_IVF], true);
+}
+
+export function searchEmbeddings(
+    database: Database<IvfEmbeddingRoot>,
+    query: StoredEmbedding,
+    options: { probeBudget: number; resultCount: number },
+): SearchHit[] | undefined {
+    const config = database.readData(root => root.config);
+    if (!config) return undefined;
+    const steps = database.readData(root => root.steps);
+    if (!steps) return undefined;
+
+    const hits: SearchHit[] = [];
+    if (!steps[STEP_IVF]) {
+        const flat = transactionRead(flatStore(database));
+        if (!flat) return undefined;
+        for (const ref of flat.keys()) {
+            hits.push({ ref, closeness: getCloseness(query, flat.get(ref)!) });
+        }
+        hits.sort((left, right) => right.closeness - left.closeness);
+        return hits.slice(0, options.resultCount);
+    }
+
+    const centroids = transactionRead(centroidStore(database));
+    if (!centroids) return undefined;
+    if (!centroids.size) return [];
+    const probeCellCount = Math.max(1, Math.ceil(options.probeBudget / config.cellTargetSize));
+    const probeCellIds = rankCellsByCloseness(query, centroids).slice(0, probeCellCount);
+    const stores = database.readData(root => probeCellIds.map(cellId => root.cells[cellId]));
+    if (!stores) return undefined;
+    for (const store of stores) {
+        const members = replayTransactionStore<StoredEmbedding>(store);
+        for (const ref of members.keys()) {
+            hits.push({ ref, closeness: getCloseness(query, members.get(ref)!) });
+        }
+    }
+    hits.sort((left, right) => right.closeness - left.closeness);
+    return hits.slice(0, options.resultCount);
+}
+
+export function insertEmbeddings(
+    database: Database<IvfEmbeddingRoot>,
+    items: EmbeddingInput[],
+) {
+    if (!items.length) return;
+    const config = database.readData(root => root.config);
+    const steps = database.readData(root => root.steps);
+    let count = database.readData(root => root.count);
+    if (!config) return;
+    if (!steps) return;
+    if (count === undefined) return;
+    const newCount = count + items.length;
+
+    if (!steps[STEP_IVF]) {
+        const flatWrites = items.map(item => ({ key: item.ref, value: item.embedding }));
+        transactionMutate(flatStore(database), flatWrites);
+        database.writeData(root => root.count, newCount);
+        if (newCount > FLAT_LIMIT) {
+            rebuildStructure(database);
+        }
+        return;
+    }
+
+    const centroids = transactionRead(centroidStore(database));
+    if (!centroids) return undefined;
+    const itemsByCell = new Map<string, EmbeddingInput[]>();
+    for (const item of items) {
+        const cellId = rankCellsByCloseness(item.embedding, centroids)[0];
+        let group = itemsByCell.get(cellId);
+        if (!group) {
+            group = [];
+            itemsByCell.set(cellId, group);
+        }
+        group.push(item);
+    }
+    for (const cellId of itemsByCell.keys()) {
+        const group = itemsByCell.get(cellId)!;
+        const memberWrites = group.map(item => ({ key: item.ref, value: item.embedding }));
+        transactionMutate(cellStore(database, cellId), memberWrites);
+    }
+    database.writeData(root => root.count, newCount);
+
+
+    const shouldRegenerate = (database: Database<IvfEmbeddingRoot>) => {
+        let steps = database.readData(root => root.steps);
+        if (!steps) return;
+        for (const threshold of REGENERATE_AT) {
+            const stepName = "regen" + threshold;
+            if (!steps[stepName] && newCount > threshold) {
+                database.writeData(root => root.steps[stepName], true);
+                return true;
+            }
+        }
+        const averageFill = newCount / Math.max(1, centroids.size) / config.cellTargetSize;
+        if (Math.random() < rebalanceProbability(averageFill)) {
+            rebuildStructure(database);
+        }
+        return true;
+    };
+    if (shouldRegenerate(database)) {
+        rebuildStructure(database);
+    }
+}
+
+export function removeEmbeddings(
+    database: Database<IvfEmbeddingRoot>,
+    items: EmbeddingInput[],
+) {
+    if (!items.length) return;
+    const steps = database.readData(root => root.steps);
+    const count = database.readData(root => root.count);
+    if (!steps) return;
+    if (count === undefined) return;
+
+    if (!steps[STEP_IVF]) {
+        const flatDeletes = items.map(item => ({ key: item.ref, value: undefined }));
+        transactionMutate(flatStore(database), flatDeletes);
+        database.writeData(root => root.count, Math.max(0, count - items.length));
+        return;
+    }
+
+    const centroids = transactionRead(centroidStore(database));
+    if (!centroids) return;
+    if (!centroids.size) return;
+
+    const candidatesByItem: { ref: string; cellIds: string[] }[] = [];
+    const candidateSet = new Set<string>();
+    for (const item of items) {
+        const cellIds = rankCellsByCloseness(item.embedding, centroids).slice(0, 1 + DELETE_FALLBACK_CELLS);
+        candidatesByItem.push({ ref: item.ref, cellIds });
+        for (const cellId of cellIds) {
+            candidateSet.add(cellId);
+        }
+    }
+
+    const candidateCellIds = Array.from(candidateSet);
+    const stores = database.readData(root => candidateCellIds.map(cellId => root.cells[cellId]));
+    if (!stores) return;
+    const membersByCell = new Map<string, Map<string, StoredEmbedding>>();
+    for (let index = 0; index < candidateCellIds.length; index++) {
+        membersByCell.set(candidateCellIds[index], replayTransactionStore(stores[index]));
+    }
+
+    const deletesByCell = new Map<string, string[]>();
+    for (const candidate of candidatesByItem) {
+        for (const cellId of candidate.cellIds) {
+            const members = membersByCell.get(cellId);
+            if (!members || !members.has(candidate.ref)) {
+                continue;
+            }
+            let refs = deletesByCell.get(cellId);
+            if (!refs) {
+                refs = [];
+                deletesByCell.set(cellId, refs);
+            }
+            refs.push(candidate.ref);
+            break;
+        }
+    }
+
+    let deletedCount = 0;
+    for (const cellId of deletesByCell.keys()) {
+        const refs = deletesByCell.get(cellId)!;
+        deletedCount += refs.length;
+        const memberDeletes = refs.map(ref => ({ key: ref, value: undefined }));
+        transactionMutate(cellStore(database, cellId), memberDeletes);
+    }
+    if (deletedCount) {
+        database.writeData(root => root.count, Math.max(0, count - deletedCount));
+    }
+}

package/storage/proxydatabase/transactionSet.d.ts

@@ -1,9 +1,14 @@
 import { Database } from "./Database";
-export type TransactionSetStore = {
+declare const valueTag: unique symbol;
+export type TransactionSetStore<Value> = {
     [fileNumber: string]: Uint8Array;
+    [valueTag]?: Value;
 };
-export declare function transactionRead<Value>(database: Database<TransactionSetStore>): Map<string, Value> | undefined;
-export declare function transactionMutate<Value>(database: Database<TransactionSetStore>, transactions: {
+export declare function transactionRead<Value>(database: Database<TransactionSetStore<Value>>): Map<string, Value> | undefined;
+export declare function replayTransactionStore<Value>(store: TransactionSetStore<Value> | undefined): Map<string, Value>;
+export declare function transactionMutate<Value>(database: Database<TransactionSetStore<Value>>, transactions: {
     key: string;
     value: Value | undefined;
-}[], compactAfterFiles?: number): true | undefined;
+}[], compactAfterFiles?: number): void;
+export declare function transactionDelete<Value>(database: Database<TransactionSetStore<Value>>): void;
+export {};