sliftutils 1.4.81 → 1.4.82

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,96 +1,206 @@
1
1
  import { Database, namespaceDatabase } from "./Database";
2
- import { TransactionSetStore, transactionRead, transactionMutate, replayTransactionStore } from "./transactionSet";
3
- import { StoredEmbedding, EmbeddingFormat, getCloseness, averageEmbeddings, hashEmbedding } from "../embeddingFormats";
2
+ import { TransactionSetStore, transactionRead, transactionMutate, transactionDelete, replayTransactionStore } from "./transactionSet";
3
+ import { StoredEmbedding, EmbeddingFormat, getCloseness, embeddingToFloat32, releaseFloat32, encodeEmbedding, hashEmbedding } from "../embeddingFormats";
4
4
 
5
5
  export type IvfConfig = {
6
6
  model: string;
7
7
  format: EmbeddingFormat;
8
- cellTarget: number;
9
- splitAt: number;
8
+ // Target number of embeddings per cell.
9
+ cellTargetSize: number;
10
10
  };
11
11
 
12
- // Cell ids ARE the hash of the cell's centroid, so there's no id counter. centroids maps that id to the
13
- // centroid embedding (read whole as the preload, ranked in RAM). Each cell is its own transaction set of
14
- // ref member embedding. refIndex maps a ref to the cell holding it, so a remove can find it.
12
+ // Tiered storage. Below FLAT_LIMIT embeddings the whole set lives in one flat transaction set (no IVF);
13
+ // once it grows past that we build the IVF (centroids + per-cell sets) and stay there forever. `count` is
14
+ // the live embedding count (re-derived exactly on a rebuild). `steps` records one-time upgrades that must
15
+ // never re-run even if the set later shrinks and regrows.
15
16
  export type IvfEmbeddingRoot = {
16
17
  config: IvfConfig;
17
- centroids: TransactionSetStore;
18
- cells: { [cellId: string]: TransactionSetStore };
19
- refIndex: TransactionSetStore;
18
+ count: number;
19
+ flat: TransactionSetStore<StoredEmbedding>;
20
+ steps: { [step: string]: boolean };
21
+ centroids: TransactionSetStore<StoredEmbedding>;
22
+ cells: { [cellId: string]: TransactionSetStore<StoredEmbedding> };
20
23
  };
21
24
 
22
25
  export type EmbeddingInput = { ref: string; embedding: StoredEmbedding };
23
26
  export type SearchHit = { ref: string; closeness: number };
27
+ type CellEntry = { ref: string; embedding: StoredEmbedding };
24
28
 
25
- function centroidStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore> {
29
+ // Below this many embeddings we skip the IVF entirely (flat store). 1024 = a clean power of two.
30
+ const FLAT_LIMIT = 1024;
31
+ // Force a from-scratch rebuild once at each of these (multiples of 4 above FLAT_LIMIT, up to ~16k), gated by a step flag so each runs at most once ever.
32
+ const REGENERATE_AT = [4096, 16384];
33
+ const STEP_IVF = "ivf";
34
+ // On delete, the member's exact cell plus this many nearby cells are checked, in case a rebuild left it non-optimal.
35
+ const DELETE_FALLBACK_CELLS = 10;
36
+ const REBALANCE_ITERATIONS = 4;
37
+
38
+ function flatStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore<StoredEmbedding>> {
39
+ return namespaceDatabase(database, root => root.flat);
40
+ }
41
+ function centroidStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore<StoredEmbedding>> {
26
42
  return namespaceDatabase(database, root => root.centroids);
27
43
  }
28
- function cellStore(database: Database<IvfEmbeddingRoot>, cellId: string): Database<TransactionSetStore> {
44
+ function cellStore(database: Database<IvfEmbeddingRoot>, cellId: string): Database<TransactionSetStore<StoredEmbedding>> {
29
45
  return namespaceDatabase(database, root => root.cells[cellId]);
30
46
  }
31
- function refIndexStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore> {
32
- return namespaceDatabase(database, root => root.refIndex);
47
+
48
+ function rankCellsByCloseness(embedding: StoredEmbedding, centroids: Map<string, StoredEmbedding>): string[] {
49
+ const ranked: { cellId: string; closeness: number }[] = [];
50
+ for (const cellId of centroids.keys()) {
51
+ ranked.push({ cellId, closeness: getCloseness(embedding, centroids.get(cellId)!) });
52
+ }
53
+ ranked.sort((left, right) => right.closeness - left.closeness);
54
+ return ranked.map(entry => entry.cellId);
33
55
  }
34
56
 
35
- function nearestCell(embedding: StoredEmbedding, centroids: Map<string, StoredEmbedding>): string {
36
- let bestCellId = "";
37
- let bestCloseness = -Infinity;
38
- for (const [cellId, centroid] of centroids) {
39
- const closeness = getCloseness(embedding, centroid);
40
- if (closeness > bestCloseness) {
41
- bestCloseness = closeness;
42
- bestCellId = cellId;
43
- }
57
+ // Per-write chance of a full rebuild. Zero at/under the target size, then rises (cubically) past it so cells
58
+ // stay roughly between target and ~2x target.
59
+ function rebalanceProbability(fillRatio: number): number {
60
+ if (fillRatio <= 1) {
61
+ return 0;
44
62
  }
45
- return bestCellId;
63
+ const over = fillRatio - 1;
64
+ return Math.min(1, over * over * over * 0.25);
46
65
  }
47
66
 
48
- // 2-means on a cell's members so an overgrown cell can split in place. Seeds with one member and the one
49
- // farthest from it, then alternates assign / recompute-centroid a few times.
50
- function splitInTwo(
51
- members: [string, StoredEmbedding][],
52
- config: IvfConfig,
53
- ): { centroid: StoredEmbedding; members: [string, StoredEmbedding][] }[] {
54
- let centroidA = members[0][1];
55
- let centroidB = members[0][1];
56
- let worst = Infinity;
57
- for (const [, embedding] of members) {
58
- const closeness = getCloseness(centroidA, embedding);
59
- if (closeness < worst) {
60
- worst = closeness;
61
- centroidB = embedding;
62
- }
67
+ // k-means. Decodes every member to a pooled float32 buffer ONCE, then assigns with a plain internal float
68
+ // dot (no getCloseness call comparing two float vectors is trivial) and keeps centroids as float means,
69
+ // encoding them to StoredEmbedding only at the end. Releases the borrowed buffers when done.
70
+ function clusterMembers(members: CellEntry[], clusterCount: number, config: IvfConfig): { centroid: StoredEmbedding; members: CellEntry[] }[] {
71
+ const memberFloats: Float32Array[] = [];
72
+ for (const member of members) {
73
+ memberFloats.push(embeddingToFloat32(member.embedding, true));
74
+ }
75
+ const length = memberFloats.length ? memberFloats[0].length : 0;
76
+ let centroids: Float32Array[] = [];
77
+ const seedStep = members.length / clusterCount;
78
+ for (let clusterIndex = 0; clusterIndex < clusterCount; clusterIndex++) {
79
+ centroids.push(new Float32Array(memberFloats[Math.floor(clusterIndex * seedStep)]));
63
80
  }
64
- let groupA: [string, StoredEmbedding][] = [];
65
- let groupB: [string, StoredEmbedding][] = [];
66
- for (let iteration = 0; iteration < 5; iteration++) {
67
- groupA = [];
68
- groupB = [];
69
- for (const member of members) {
70
- if (getCloseness(centroidA, member[1]) >= getCloseness(centroidB, member[1])) {
71
- groupA.push(member);
72
- } else {
73
- groupB.push(member);
81
+ let groups: number[][] = [];
82
+ for (let iteration = 0; iteration < REBALANCE_ITERATIONS; iteration++) {
83
+ groups = [];
84
+ for (let clusterIndex = 0; clusterIndex < centroids.length; clusterIndex++) {
85
+ groups.push([]);
86
+ }
87
+ for (let memberIndex = 0; memberIndex < memberFloats.length; memberIndex++) {
88
+ const memberFloat = memberFloats[memberIndex];
89
+ let bestIndex = 0;
90
+ let bestDot = -Infinity;
91
+ for (let centroidIndex = 0; centroidIndex < centroids.length; centroidIndex++) {
92
+ const centroidFloat = centroids[centroidIndex];
93
+ let dot = 0;
94
+ for (let dim = 0; dim < length; dim++) {
95
+ dot += memberFloat[dim] * centroidFloat[dim];
96
+ }
97
+ if (dot > bestDot) {
98
+ bestDot = dot;
99
+ bestIndex = centroidIndex;
100
+ }
74
101
  }
102
+ groups[bestIndex].push(memberIndex);
75
103
  }
76
- if (!groupA.length || !groupB.length) {
77
- break;
104
+ const nextCentroids: Float32Array[] = [];
105
+ const nextGroups: number[][] = [];
106
+ for (const group of groups) {
107
+ if (!group.length) {
108
+ continue;
109
+ }
110
+ const mean = new Float32Array(length);
111
+ for (const memberIndex of group) {
112
+ const memberFloat = memberFloats[memberIndex];
113
+ for (let dim = 0; dim < length; dim++) {
114
+ mean[dim] += memberFloat[dim];
115
+ }
116
+ }
117
+ let norm = 0;
118
+ for (let dim = 0; dim < length; dim++) {
119
+ norm += mean[dim] * mean[dim];
120
+ }
121
+ const magnitude = Math.sqrt(norm) || 1;
122
+ for (let dim = 0; dim < length; dim++) {
123
+ mean[dim] /= magnitude;
124
+ }
125
+ nextCentroids.push(mean);
126
+ nextGroups.push(group);
78
127
  }
79
- centroidA = averageEmbeddings(groupA.map(member => member[1]), config);
80
- centroidB = averageEmbeddings(groupB.map(member => member[1]), config);
128
+ centroids = nextCentroids;
129
+ groups = nextGroups;
81
130
  }
82
- const result: { centroid: StoredEmbedding; members: [string, StoredEmbedding][] }[] = [];
83
- if (groupA.length) {
84
- result.push({ centroid: centroidA, members: groupA });
131
+ const result: { centroid: StoredEmbedding; members: CellEntry[] }[] = [];
132
+ for (let clusterIndex = 0; clusterIndex < centroids.length; clusterIndex++) {
133
+ const centroid = encodeEmbedding({ input: centroids[clusterIndex], format: config.format, model: config.model });
134
+ const cellMembers: CellEntry[] = [];
135
+ for (const memberIndex of groups[clusterIndex]) {
136
+ cellMembers.push(members[memberIndex]);
137
+ }
138
+ result.push({ centroid, members: cellMembers });
85
139
  }
86
- if (groupB.length) {
87
- result.push({ centroid: centroidB, members: groupB });
140
+ for (const memberFloat of memberFloats) {
141
+ releaseFloat32(memberFloat);
88
142
  }
89
143
  return result;
90
144
  }
91
145
 
92
- // Nearest cells until probeBudget members are covered, scored exactly against the query. undefined while
93
- // not synced. All probed cells are fetched in one batched read so probing doesn't cascade per cell.
146
+ // Re-cluster EVERY embedding (flat tier + all cells) into a fresh IVF, clear the flat tier, and mark IVF
147
+ // mode. This is the rearranger: run probabilistically as the set grows, once at each regenerate threshold,
148
+ // and once to upgrade from flat. Reads everything it uses, so it bails (does nothing) if anything isn't
149
+ // synced and the caller's retry re-runs it; no caller cares about its result.
150
+ export function rebuildStructure(database: Database<IvfEmbeddingRoot>): void {
151
+ const config = database.readData(root => root.config);
152
+ if (!config) return;
153
+ const steps = database.readData(root => root.steps);
154
+ if (!steps) return;
155
+
156
+ const allMembers: CellEntry[] = [];
157
+ // Only the one-time flat->IVF upgrade pulls from the flat tier. Once STEP_IVF is set the flat data has
158
+ // already been folded into cells, so later rebuilds ignore it. It's left in place, never deleted: deleteData
159
+ // only removes primitive leaves, not a whole store, and the flat tier is small enough not to matter.
160
+ if (!steps[STEP_IVF]) {
161
+ const flat = transactionRead(flatStore(database));
162
+ if (!flat) return;
163
+ for (const ref of flat.keys()) {
164
+ allMembers.push({ ref, embedding: flat.get(ref)! });
165
+ }
166
+ }
167
+ const centroids = transactionRead(centroidStore(database));
168
+ if (!centroids) return;
169
+ const oldCellIds = Array.from(centroids.keys());
170
+ const cellStores = database.readData(root => oldCellIds.map(cellId => root.cells[cellId]));
171
+ if (!cellStores) return;
172
+ for (const store of cellStores) {
173
+ const members = replayTransactionStore<StoredEmbedding>(store);
174
+ for (const ref of members.keys()) {
175
+ allMembers.push({ ref, embedding: members.get(ref)! });
176
+ }
177
+ }
178
+ if (!allMembers.length) return;
179
+
180
+ const clusterCount = Math.max(1, Math.round(allMembers.length / config.cellTargetSize));
181
+ const clusters = clusterMembers(allMembers, clusterCount, config);
182
+
183
+ const newCellIds = new Set<string>();
184
+ const centroidWrites: { key: string; value: StoredEmbedding | undefined }[] = [];
185
+ for (const cluster of clusters) {
186
+ const cellId = hashEmbedding(cluster.centroid);
187
+ newCellIds.add(cellId);
188
+ centroidWrites.push({ key: cellId, value: cluster.centroid });
189
+ const memberWrites = cluster.members.map(member => ({ key: member.ref, value: member.embedding }));
190
+ transactionMutate(cellStore(database, cellId), memberWrites);
191
+ }
192
+ for (const oldCellId of oldCellIds) {
193
+ if (newCellIds.has(oldCellId)) {
194
+ continue;
195
+ }
196
+ centroidWrites.push({ key: oldCellId, value: undefined });
197
+ transactionDelete(cellStore(database, oldCellId));
198
+ }
199
+ transactionMutate(centroidStore(database), centroidWrites);
200
+ database.writeData(root => root.count, allMembers.length);
201
+ database.writeData(root => root.steps[STEP_IVF], true);
202
+ }
203
+
94
204
  export function searchEmbeddings(
95
205
  database: Database<IvfEmbeddingRoot>,
96
206
  query: StoredEmbedding,
@@ -98,54 +208,65 @@ export function searchEmbeddings(
98
208
  ): SearchHit[] | undefined {
99
209
  const config = database.readData(root => root.config);
100
210
  if (!config) return undefined;
101
- const centroids = transactionRead<StoredEmbedding>(centroidStore(database));
102
- if (!centroids) return undefined;
103
- if (!centroids.size) return [];
211
+ const steps = database.readData(root => root.steps);
212
+ if (!steps) return undefined;
104
213
 
105
- const rankedCellIds = [...centroids.entries()]
106
- .map(([cellId, centroid]) => ({ cellId, closeness: getCloseness(query, centroid) }))
107
- .sort((left, right) => right.closeness - left.closeness)
108
- .map(ranked => ranked.cellId);
214
+ const hits: SearchHit[] = [];
215
+ if (!steps[STEP_IVF]) {
216
+ const flat = transactionRead(flatStore(database));
217
+ if (!flat) return undefined;
218
+ for (const ref of flat.keys()) {
219
+ hits.push({ ref, closeness: getCloseness(query, flat.get(ref)!) });
220
+ }
221
+ hits.sort((left, right) => right.closeness - left.closeness);
222
+ return hits.slice(0, options.resultCount);
223
+ }
109
224
 
110
- const probeCellCount = Math.max(1, Math.ceil(options.probeBudget / config.cellTarget));
111
- const probeCellIds = rankedCellIds.slice(0, probeCellCount);
225
+ const centroids = transactionRead(centroidStore(database));
226
+ if (!centroids) return undefined;
227
+ if (!centroids.size) return [];
228
+ const probeCellCount = Math.max(1, Math.ceil(options.probeBudget / config.cellTargetSize));
229
+ const probeCellIds = rankCellsByCloseness(query, centroids).slice(0, probeCellCount);
112
230
  const stores = database.readData(root => probeCellIds.map(cellId => root.cells[cellId]));
113
231
  if (!stores) return undefined;
114
-
115
- const hits: SearchHit[] = [];
116
232
  for (const store of stores) {
117
233
  const members = replayTransactionStore<StoredEmbedding>(store);
118
- for (const [ref, embedding] of members) {
119
- hits.push({ ref, closeness: getCloseness(query, embedding) });
234
+ for (const ref of members.keys()) {
235
+ hits.push({ ref, closeness: getCloseness(query, members.get(ref)!) });
120
236
  }
121
237
  }
122
238
  hits.sort((left, right) => right.closeness - left.closeness);
123
239
  return hits.slice(0, options.resultCount);
124
240
  }
125
241
 
126
- // Assign each embedding to its nearest cell and add it. An empty index bootstraps a first cell from the
127
- // inserted batch; a cell that crosses splitAt*cellTarget splits in place. undefined while not synced.
128
242
  export function insertEmbeddings(
129
243
  database: Database<IvfEmbeddingRoot>,
130
244
  items: EmbeddingInput[],
131
- ): true | undefined {
132
- if (!items.length) return true;
245
+ ) {
246
+ if (!items.length) return;
133
247
  const config = database.readData(root => root.config);
134
- if (!config) return undefined;
135
- const centroids = transactionRead<StoredEmbedding>(centroidStore(database));
136
- if (!centroids) return undefined;
248
+ const steps = database.readData(root => root.steps);
249
+ let count = database.readData(root => root.count);
250
+ if (!config) return;
251
+ if (!steps) return;
252
+ if (count === undefined) return;
253
+ const newCount = count + items.length;
137
254
 
138
- const newCentroids: { key: string; value: StoredEmbedding | undefined }[] = [];
139
- if (!centroids.size) {
140
- const centroid = averageEmbeddings(items.map(item => item.embedding), config);
141
- const cellId = hashEmbedding(centroid);
142
- centroids.set(cellId, centroid);
143
- newCentroids.push({ key: cellId, value: centroid });
255
+ if (!steps[STEP_IVF]) {
256
+ const flatWrites = items.map(item => ({ key: item.ref, value: item.embedding }));
257
+ transactionMutate(flatStore(database), flatWrites);
258
+ database.writeData(root => root.count, newCount);
259
+ if (newCount > FLAT_LIMIT) {
260
+ rebuildStructure(database);
261
+ }
262
+ return;
144
263
  }
145
264
 
265
+ const centroids = transactionRead(centroidStore(database));
266
+ if (!centroids) return undefined;
146
267
  const itemsByCell = new Map<string, EmbeddingInput[]>();
147
268
  for (const item of items) {
148
- const cellId = nearestCell(item.embedding, centroids);
269
+ const cellId = rankCellsByCloseness(item.embedding, centroids)[0];
149
270
  let group = itemsByCell.get(cellId);
150
271
  if (!group) {
151
272
  group = [];
@@ -153,75 +274,99 @@ export function insertEmbeddings(
153
274
  }
154
275
  group.push(item);
155
276
  }
156
-
157
- const targetCellIds = [...itemsByCell.keys()];
158
- const existingStores = database.readData(root => targetCellIds.map(cellId => root.cells[cellId]));
159
- if (!existingStores) return undefined;
160
-
161
- const splitThreshold = config.splitAt * config.cellTarget;
162
- const refWrites: { key: string; value: string | undefined }[] = [];
163
- for (let cellIndex = 0; cellIndex < targetCellIds.length; cellIndex++) {
164
- const cellId = targetCellIds[cellIndex];
277
+ for (const cellId of itemsByCell.keys()) {
165
278
  const group = itemsByCell.get(cellId)!;
166
- const combined = replayTransactionStore<StoredEmbedding>(existingStores[cellIndex]);
167
- for (const item of group) {
168
- combined.set(item.ref, item.embedding);
169
- }
279
+ const memberWrites = group.map(item => ({ key: item.ref, value: item.embedding }));
280
+ transactionMutate(cellStore(database, cellId), memberWrites);
281
+ }
282
+ database.writeData(root => root.count, newCount);
170
283
 
171
- if (combined.size <= splitThreshold) {
172
- const memberWrites = group.map(item => ({ key: item.ref, value: item.embedding }));
173
- if (!transactionMutate(cellStore(database, cellId), memberWrites)) return undefined;
174
- for (const item of group) {
175
- refWrites.push({ key: item.ref, value: cellId });
176
- }
177
- continue;
178
- }
179
284
 
180
- const subCells = splitInTwo([...combined.entries()], config);
181
- for (const subCell of subCells) {
182
- const subCellId = hashEmbedding(subCell.centroid);
183
- newCentroids.push({ key: subCellId, value: subCell.centroid });
184
- const memberWrites = subCell.members.map(([ref, embedding]) => ({ key: ref, value: embedding }));
185
- if (!transactionMutate(cellStore(database, subCellId), memberWrites)) return undefined;
186
- for (const [ref] of subCell.members) {
187
- refWrites.push({ key: ref, value: subCellId });
285
+ const shouldRegenerate = (database: Database<IvfEmbeddingRoot>) => {
286
+ let steps = database.readData(root => root.steps);
287
+ if (!steps) return;
288
+ for (const threshold of REGENERATE_AT) {
289
+ const stepName = "regen" + threshold;
290
+ if (!steps[stepName] && newCount > threshold) {
291
+ database.writeData(root => root.steps[stepName], true);
292
+ return true;
188
293
  }
189
294
  }
190
- newCentroids.push({ key: cellId, value: undefined });
191
- database.deleteData(root => root.cells[cellId]);
295
+ const averageFill = newCount / Math.max(1, centroids.size) / config.cellTargetSize;
296
+ if (Math.random() < rebalanceProbability(averageFill)) {
297
+ rebuildStructure(database);
298
+ }
299
+ return true;
300
+ };
301
+ if (shouldRegenerate(database)) {
302
+ rebuildStructure(database);
192
303
  }
193
-
194
- if (newCentroids.length && !transactionMutate(centroidStore(database), newCentroids)) return undefined;
195
- if (refWrites.length && !transactionMutate(refIndexStore(database), refWrites)) return undefined;
196
- return true;
197
304
  }
198
305
 
199
- // Tombstone each ref in its cell (found via refIndex) and drop it from the index. undefined while not synced.
200
306
  export function removeEmbeddings(
201
307
  database: Database<IvfEmbeddingRoot>,
202
- refs: string[],
203
- ): true | undefined {
204
- if (!refs.length) return true;
205
- const refIndex = transactionRead<string>(refIndexStore(database));
206
- if (!refIndex) return undefined;
207
-
208
- const refsByCell = new Map<string, string[]>();
209
- for (const ref of refs) {
210
- const cellId = refIndex.get(ref);
211
- if (!cellId) continue;
212
- let group = refsByCell.get(cellId);
213
- if (!group) {
214
- group = [];
215
- refsByCell.set(cellId, group);
308
+ items: EmbeddingInput[],
309
+ ) {
310
+ if (!items.length) return;
311
+ const steps = database.readData(root => root.steps);
312
+ const count = database.readData(root => root.count);
313
+ if (!steps) return;
314
+ if (count === undefined) return;
315
+
316
+ if (!steps[STEP_IVF]) {
317
+ const flatDeletes = items.map(item => ({ key: item.ref, value: undefined }));
318
+ transactionMutate(flatStore(database), flatDeletes);
319
+ database.writeData(root => root.count, Math.max(0, count - items.length));
320
+ return;
321
+ }
322
+
323
+ const centroids = transactionRead(centroidStore(database));
324
+ if (!centroids) return;
325
+ if (!centroids.size) return;
326
+
327
+ const candidatesByItem: { ref: string; cellIds: string[] }[] = [];
328
+ const candidateSet = new Set<string>();
329
+ for (const item of items) {
330
+ const cellIds = rankCellsByCloseness(item.embedding, centroids).slice(0, 1 + DELETE_FALLBACK_CELLS);
331
+ candidatesByItem.push({ ref: item.ref, cellIds });
332
+ for (const cellId of cellIds) {
333
+ candidateSet.add(cellId);
216
334
  }
217
- group.push(ref);
218
335
  }
219
336
 
220
- for (const [cellId, group] of refsByCell) {
221
- const memberDeletes = group.map(ref => ({ key: ref, value: undefined }));
222
- if (!transactionMutate<StoredEmbedding>(cellStore(database, cellId), memberDeletes)) return undefined;
337
+ const candidateCellIds = Array.from(candidateSet);
338
+ const stores = database.readData(root => candidateCellIds.map(cellId => root.cells[cellId]));
339
+ if (!stores) return;
340
+ const membersByCell = new Map<string, Map<string, StoredEmbedding>>();
341
+ for (let index = 0; index < candidateCellIds.length; index++) {
342
+ membersByCell.set(candidateCellIds[index], replayTransactionStore(stores[index]));
343
+ }
344
+
345
+ const deletesByCell = new Map<string, string[]>();
346
+ for (const candidate of candidatesByItem) {
347
+ for (const cellId of candidate.cellIds) {
348
+ const members = membersByCell.get(cellId);
349
+ if (!members || !members.has(candidate.ref)) {
350
+ continue;
351
+ }
352
+ let refs = deletesByCell.get(cellId);
353
+ if (!refs) {
354
+ refs = [];
355
+ deletesByCell.set(cellId, refs);
356
+ }
357
+ refs.push(candidate.ref);
358
+ break;
359
+ }
360
+ }
361
+
362
+ let deletedCount = 0;
363
+ for (const cellId of deletesByCell.keys()) {
364
+ const refs = deletesByCell.get(cellId)!;
365
+ deletedCount += refs.length;
366
+ const memberDeletes = refs.map(ref => ({ key: ref, value: undefined }));
367
+ transactionMutate(cellStore(database, cellId), memberDeletes);
368
+ }
369
+ if (deletedCount) {
370
+ database.writeData(root => root.count, Math.max(0, count - deletedCount));
223
371
  }
224
- const refDeletes = refs.map(ref => ({ key: ref, value: undefined }));
225
- if (!transactionMutate<string>(refIndexStore(database), refDeletes)) return undefined;
226
- return true;
227
372
  }
@@ -1,10 +1,14 @@
1
1
  import { Database } from "./Database";
2
- export type TransactionSetStore = {
2
+ declare const valueTag: unique symbol;
3
+ export type TransactionSetStore<Value> = {
3
4
  [fileNumber: string]: Uint8Array;
5
+ [valueTag]?: Value;
4
6
  };
5
- export declare function transactionRead<Value>(database: Database<TransactionSetStore>): Map<string, Value> | undefined;
6
- export declare function replayTransactionStore<Value>(store: TransactionSetStore | undefined): Map<string, Value>;
7
- export declare function transactionMutate<Value>(database: Database<TransactionSetStore>, transactions: {
7
+ export declare function transactionRead<Value>(database: Database<TransactionSetStore<Value>>): Map<string, Value> | undefined;
8
+ export declare function replayTransactionStore<Value>(store: TransactionSetStore<Value> | undefined): Map<string, Value>;
9
+ export declare function transactionMutate<Value>(database: Database<TransactionSetStore<Value>>, transactions: {
8
10
  key: string;
9
11
  value: Value | undefined;
10
- }[], compactAfterFiles?: number): true | undefined;
12
+ }[], compactAfterFiles?: number): void;
13
+ export declare function transactionDelete<Value>(database: Database<TransactionSetStore<Value>>): void;
14
+ export {};