sliftutils 1.4.81 → 1.4.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +61 -13
- package/package.json +1 -1
- package/spec.txt +18 -0
- package/storage/dist/embeddingFormats.ts.cache +340 -0
- package/storage/dist/faceFramesData.ts.cache +55 -0
- package/storage/embeddingBench.d.ts +1 -0
- package/storage/embeddingBench.ts +152 -0
- package/storage/embeddingFormats.d.ts +4 -1
- package/storage/embeddingFormats.ts +173 -111
- package/storage/faceFramesData.d.ts +1 -0
- package/storage/faceFramesData.ts +49 -0
- package/storage/faceFramesServer.d.ts +1 -0
- package/storage/faceFramesServer.ts +56 -0
- package/storage/proxydatabase/dist/Database.ts.cache +13 -0
- package/storage/proxydatabase/dist/inMemoryDatabase.ts.cache +105 -0
- package/storage/proxydatabase/dist/ivfDbCheck.ts.cache +81 -0
- package/storage/proxydatabase/dist/ivfEmbeddingDatabase.ts.cache +334 -0
- package/storage/proxydatabase/dist/transactionSet.ts.cache +108 -0
- package/storage/proxydatabase/inMemoryDatabase.d.ts +13 -0
- package/storage/proxydatabase/inMemoryDatabase.ts +108 -0
- package/storage/proxydatabase/ivfDbCheck.d.ts +1 -0
- package/storage/proxydatabase/ivfDbCheck.ts +85 -0
- package/storage/proxydatabase/ivfEmbeddingDatabase.d.ts +11 -7
- package/storage/proxydatabase/ivfEmbeddingDatabase.ts +289 -144
- package/storage/proxydatabase/transactionSet.d.ts +9 -5
- package/storage/proxydatabase/transactionSet.ts +43 -22
|
@@ -1,96 +1,206 @@
|
|
|
1
1
|
import { Database, namespaceDatabase } from "./Database";
|
|
2
|
-
import { TransactionSetStore, transactionRead, transactionMutate, replayTransactionStore } from "./transactionSet";
|
|
3
|
-
import { StoredEmbedding, EmbeddingFormat, getCloseness,
|
|
2
|
+
import { TransactionSetStore, transactionRead, transactionMutate, transactionDelete, replayTransactionStore } from "./transactionSet";
|
|
3
|
+
import { StoredEmbedding, EmbeddingFormat, getCloseness, embeddingToFloat32, releaseFloat32, encodeEmbedding, hashEmbedding } from "../embeddingFormats";
|
|
4
4
|
|
|
5
5
|
export type IvfConfig = {
|
|
6
6
|
model: string;
|
|
7
7
|
format: EmbeddingFormat;
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
// Target number of embeddings per cell.
|
|
9
|
+
cellTargetSize: number;
|
|
10
10
|
};
|
|
11
11
|
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
12
|
+
// Tiered storage. Below FLAT_LIMIT embeddings the whole set lives in one flat transaction set (no IVF);
|
|
13
|
+
// once it grows past that we build the IVF (centroids + per-cell sets) and stay there forever. `count` is
|
|
14
|
+
// the live embedding count (re-derived exactly on a rebuild). `steps` records one-time upgrades that must
|
|
15
|
+
// never re-run even if the set later shrinks and regrows.
|
|
15
16
|
export type IvfEmbeddingRoot = {
|
|
16
17
|
config: IvfConfig;
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
count: number;
|
|
19
|
+
flat: TransactionSetStore<StoredEmbedding>;
|
|
20
|
+
steps: { [step: string]: boolean };
|
|
21
|
+
centroids: TransactionSetStore<StoredEmbedding>;
|
|
22
|
+
cells: { [cellId: string]: TransactionSetStore<StoredEmbedding> };
|
|
20
23
|
};
|
|
21
24
|
|
|
22
25
|
export type EmbeddingInput = { ref: string; embedding: StoredEmbedding };
|
|
23
26
|
export type SearchHit = { ref: string; closeness: number };
|
|
27
|
+
type CellEntry = { ref: string; embedding: StoredEmbedding };
|
|
24
28
|
|
|
25
|
-
|
|
29
|
+
// Below this many embeddings we skip the IVF entirely (flat store). 1024 = a clean power of two.
|
|
30
|
+
const FLAT_LIMIT = 1024;
|
|
31
|
+
// Force a from-scratch rebuild once at each of these (multiples of 4 above FLAT_LIMIT, up to ~16k), gated by a step flag so each runs at most once ever.
|
|
32
|
+
const REGENERATE_AT = [4096, 16384];
|
|
33
|
+
const STEP_IVF = "ivf";
|
|
34
|
+
// On delete, the member's exact cell plus this many nearby cells are checked, in case a rebuild left it non-optimal.
|
|
35
|
+
const DELETE_FALLBACK_CELLS = 10;
|
|
36
|
+
const REBALANCE_ITERATIONS = 4;
|
|
37
|
+
|
|
38
|
+
function flatStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore<StoredEmbedding>> {
|
|
39
|
+
return namespaceDatabase(database, root => root.flat);
|
|
40
|
+
}
|
|
41
|
+
function centroidStore(database: Database<IvfEmbeddingRoot>): Database<TransactionSetStore<StoredEmbedding>> {
|
|
26
42
|
return namespaceDatabase(database, root => root.centroids);
|
|
27
43
|
}
|
|
28
|
-
function cellStore(database: Database<IvfEmbeddingRoot>, cellId: string): Database<TransactionSetStore
|
|
44
|
+
function cellStore(database: Database<IvfEmbeddingRoot>, cellId: string): Database<TransactionSetStore<StoredEmbedding>> {
|
|
29
45
|
return namespaceDatabase(database, root => root.cells[cellId]);
|
|
30
46
|
}
|
|
31
|
-
|
|
32
|
-
|
|
47
|
+
|
|
48
|
+
function rankCellsByCloseness(embedding: StoredEmbedding, centroids: Map<string, StoredEmbedding>): string[] {
|
|
49
|
+
const ranked: { cellId: string; closeness: number }[] = [];
|
|
50
|
+
for (const cellId of centroids.keys()) {
|
|
51
|
+
ranked.push({ cellId, closeness: getCloseness(embedding, centroids.get(cellId)!) });
|
|
52
|
+
}
|
|
53
|
+
ranked.sort((left, right) => right.closeness - left.closeness);
|
|
54
|
+
return ranked.map(entry => entry.cellId);
|
|
33
55
|
}
|
|
34
56
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
if (closeness > bestCloseness) {
|
|
41
|
-
bestCloseness = closeness;
|
|
42
|
-
bestCellId = cellId;
|
|
43
|
-
}
|
|
57
|
+
// Per-write chance of a full rebuild. Zero at/under the target size, then rises (cubically) past it so cells
|
|
58
|
+
// stay roughly between target and ~2x target.
|
|
59
|
+
function rebalanceProbability(fillRatio: number): number {
|
|
60
|
+
if (fillRatio <= 1) {
|
|
61
|
+
return 0;
|
|
44
62
|
}
|
|
45
|
-
|
|
63
|
+
const over = fillRatio - 1;
|
|
64
|
+
return Math.min(1, over * over * over * 0.25);
|
|
46
65
|
}
|
|
47
66
|
|
|
48
|
-
//
|
|
49
|
-
//
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
centroidB = embedding;
|
|
62
|
-
}
|
|
67
|
+
// k-means. Decodes every member to a pooled float32 buffer ONCE, then assigns with a plain internal float
|
|
68
|
+
// dot (no getCloseness call — comparing two float vectors is trivial) and keeps centroids as float means,
|
|
69
|
+
// encoding them to StoredEmbedding only at the end. Releases the borrowed buffers when done.
|
|
70
|
+
function clusterMembers(members: CellEntry[], clusterCount: number, config: IvfConfig): { centroid: StoredEmbedding; members: CellEntry[] }[] {
|
|
71
|
+
const memberFloats: Float32Array[] = [];
|
|
72
|
+
for (const member of members) {
|
|
73
|
+
memberFloats.push(embeddingToFloat32(member.embedding, true));
|
|
74
|
+
}
|
|
75
|
+
const length = memberFloats.length ? memberFloats[0].length : 0;
|
|
76
|
+
let centroids: Float32Array[] = [];
|
|
77
|
+
const seedStep = members.length / clusterCount;
|
|
78
|
+
for (let clusterIndex = 0; clusterIndex < clusterCount; clusterIndex++) {
|
|
79
|
+
centroids.push(new Float32Array(memberFloats[Math.floor(clusterIndex * seedStep)]));
|
|
63
80
|
}
|
|
64
|
-
let
|
|
65
|
-
let
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
81
|
+
let groups: number[][] = [];
|
|
82
|
+
for (let iteration = 0; iteration < REBALANCE_ITERATIONS; iteration++) {
|
|
83
|
+
groups = [];
|
|
84
|
+
for (let clusterIndex = 0; clusterIndex < centroids.length; clusterIndex++) {
|
|
85
|
+
groups.push([]);
|
|
86
|
+
}
|
|
87
|
+
for (let memberIndex = 0; memberIndex < memberFloats.length; memberIndex++) {
|
|
88
|
+
const memberFloat = memberFloats[memberIndex];
|
|
89
|
+
let bestIndex = 0;
|
|
90
|
+
let bestDot = -Infinity;
|
|
91
|
+
for (let centroidIndex = 0; centroidIndex < centroids.length; centroidIndex++) {
|
|
92
|
+
const centroidFloat = centroids[centroidIndex];
|
|
93
|
+
let dot = 0;
|
|
94
|
+
for (let dim = 0; dim < length; dim++) {
|
|
95
|
+
dot += memberFloat[dim] * centroidFloat[dim];
|
|
96
|
+
}
|
|
97
|
+
if (dot > bestDot) {
|
|
98
|
+
bestDot = dot;
|
|
99
|
+
bestIndex = centroidIndex;
|
|
100
|
+
}
|
|
74
101
|
}
|
|
102
|
+
groups[bestIndex].push(memberIndex);
|
|
75
103
|
}
|
|
76
|
-
|
|
77
|
-
|
|
104
|
+
const nextCentroids: Float32Array[] = [];
|
|
105
|
+
const nextGroups: number[][] = [];
|
|
106
|
+
for (const group of groups) {
|
|
107
|
+
if (!group.length) {
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
const mean = new Float32Array(length);
|
|
111
|
+
for (const memberIndex of group) {
|
|
112
|
+
const memberFloat = memberFloats[memberIndex];
|
|
113
|
+
for (let dim = 0; dim < length; dim++) {
|
|
114
|
+
mean[dim] += memberFloat[dim];
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
let norm = 0;
|
|
118
|
+
for (let dim = 0; dim < length; dim++) {
|
|
119
|
+
norm += mean[dim] * mean[dim];
|
|
120
|
+
}
|
|
121
|
+
const magnitude = Math.sqrt(norm) || 1;
|
|
122
|
+
for (let dim = 0; dim < length; dim++) {
|
|
123
|
+
mean[dim] /= magnitude;
|
|
124
|
+
}
|
|
125
|
+
nextCentroids.push(mean);
|
|
126
|
+
nextGroups.push(group);
|
|
78
127
|
}
|
|
79
|
-
|
|
80
|
-
|
|
128
|
+
centroids = nextCentroids;
|
|
129
|
+
groups = nextGroups;
|
|
81
130
|
}
|
|
82
|
-
const result: { centroid: StoredEmbedding; members: [
|
|
83
|
-
|
|
84
|
-
|
|
131
|
+
const result: { centroid: StoredEmbedding; members: CellEntry[] }[] = [];
|
|
132
|
+
for (let clusterIndex = 0; clusterIndex < centroids.length; clusterIndex++) {
|
|
133
|
+
const centroid = encodeEmbedding({ input: centroids[clusterIndex], format: config.format, model: config.model });
|
|
134
|
+
const cellMembers: CellEntry[] = [];
|
|
135
|
+
for (const memberIndex of groups[clusterIndex]) {
|
|
136
|
+
cellMembers.push(members[memberIndex]);
|
|
137
|
+
}
|
|
138
|
+
result.push({ centroid, members: cellMembers });
|
|
85
139
|
}
|
|
86
|
-
|
|
87
|
-
|
|
140
|
+
for (const memberFloat of memberFloats) {
|
|
141
|
+
releaseFloat32(memberFloat);
|
|
88
142
|
}
|
|
89
143
|
return result;
|
|
90
144
|
}
|
|
91
145
|
|
|
92
|
-
//
|
|
93
|
-
//
|
|
146
|
+
// Re-cluster EVERY embedding (flat tier + all cells) into a fresh IVF, clear the flat tier, and mark IVF
|
|
147
|
+
// mode. This is the rearranger: run probabilistically as the set grows, once at each regenerate threshold,
|
|
148
|
+
// and once to upgrade from flat. Reads everything it uses, so it bails (does nothing) if anything isn't
|
|
149
|
+
// synced and the caller's retry re-runs it; no caller cares about its result.
|
|
150
|
+
export function rebuildStructure(database: Database<IvfEmbeddingRoot>): void {
|
|
151
|
+
const config = database.readData(root => root.config);
|
|
152
|
+
if (!config) return;
|
|
153
|
+
const steps = database.readData(root => root.steps);
|
|
154
|
+
if (!steps) return;
|
|
155
|
+
|
|
156
|
+
const allMembers: CellEntry[] = [];
|
|
157
|
+
// Only the one-time flat->IVF upgrade pulls from the flat tier. Once STEP_IVF is set the flat data has
|
|
158
|
+
// already been folded into cells, so later rebuilds ignore it. It's left in place, never deleted: deleteData
|
|
159
|
+
// only removes primitive leaves, not a whole store, and the flat tier is small enough not to matter.
|
|
160
|
+
if (!steps[STEP_IVF]) {
|
|
161
|
+
const flat = transactionRead(flatStore(database));
|
|
162
|
+
if (!flat) return;
|
|
163
|
+
for (const ref of flat.keys()) {
|
|
164
|
+
allMembers.push({ ref, embedding: flat.get(ref)! });
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
const centroids = transactionRead(centroidStore(database));
|
|
168
|
+
if (!centroids) return;
|
|
169
|
+
const oldCellIds = Array.from(centroids.keys());
|
|
170
|
+
const cellStores = database.readData(root => oldCellIds.map(cellId => root.cells[cellId]));
|
|
171
|
+
if (!cellStores) return;
|
|
172
|
+
for (const store of cellStores) {
|
|
173
|
+
const members = replayTransactionStore<StoredEmbedding>(store);
|
|
174
|
+
for (const ref of members.keys()) {
|
|
175
|
+
allMembers.push({ ref, embedding: members.get(ref)! });
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
if (!allMembers.length) return;
|
|
179
|
+
|
|
180
|
+
const clusterCount = Math.max(1, Math.round(allMembers.length / config.cellTargetSize));
|
|
181
|
+
const clusters = clusterMembers(allMembers, clusterCount, config);
|
|
182
|
+
|
|
183
|
+
const newCellIds = new Set<string>();
|
|
184
|
+
const centroidWrites: { key: string; value: StoredEmbedding | undefined }[] = [];
|
|
185
|
+
for (const cluster of clusters) {
|
|
186
|
+
const cellId = hashEmbedding(cluster.centroid);
|
|
187
|
+
newCellIds.add(cellId);
|
|
188
|
+
centroidWrites.push({ key: cellId, value: cluster.centroid });
|
|
189
|
+
const memberWrites = cluster.members.map(member => ({ key: member.ref, value: member.embedding }));
|
|
190
|
+
transactionMutate(cellStore(database, cellId), memberWrites);
|
|
191
|
+
}
|
|
192
|
+
for (const oldCellId of oldCellIds) {
|
|
193
|
+
if (newCellIds.has(oldCellId)) {
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
centroidWrites.push({ key: oldCellId, value: undefined });
|
|
197
|
+
transactionDelete(cellStore(database, oldCellId));
|
|
198
|
+
}
|
|
199
|
+
transactionMutate(centroidStore(database), centroidWrites);
|
|
200
|
+
database.writeData(root => root.count, allMembers.length);
|
|
201
|
+
database.writeData(root => root.steps[STEP_IVF], true);
|
|
202
|
+
}
|
|
203
|
+
|
|
94
204
|
export function searchEmbeddings(
|
|
95
205
|
database: Database<IvfEmbeddingRoot>,
|
|
96
206
|
query: StoredEmbedding,
|
|
@@ -98,54 +208,65 @@ export function searchEmbeddings(
|
|
|
98
208
|
): SearchHit[] | undefined {
|
|
99
209
|
const config = database.readData(root => root.config);
|
|
100
210
|
if (!config) return undefined;
|
|
101
|
-
const
|
|
102
|
-
if (!
|
|
103
|
-
if (!centroids.size) return [];
|
|
211
|
+
const steps = database.readData(root => root.steps);
|
|
212
|
+
if (!steps) return undefined;
|
|
104
213
|
|
|
105
|
-
const
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
214
|
+
const hits: SearchHit[] = [];
|
|
215
|
+
if (!steps[STEP_IVF]) {
|
|
216
|
+
const flat = transactionRead(flatStore(database));
|
|
217
|
+
if (!flat) return undefined;
|
|
218
|
+
for (const ref of flat.keys()) {
|
|
219
|
+
hits.push({ ref, closeness: getCloseness(query, flat.get(ref)!) });
|
|
220
|
+
}
|
|
221
|
+
hits.sort((left, right) => right.closeness - left.closeness);
|
|
222
|
+
return hits.slice(0, options.resultCount);
|
|
223
|
+
}
|
|
109
224
|
|
|
110
|
-
const
|
|
111
|
-
|
|
225
|
+
const centroids = transactionRead(centroidStore(database));
|
|
226
|
+
if (!centroids) return undefined;
|
|
227
|
+
if (!centroids.size) return [];
|
|
228
|
+
const probeCellCount = Math.max(1, Math.ceil(options.probeBudget / config.cellTargetSize));
|
|
229
|
+
const probeCellIds = rankCellsByCloseness(query, centroids).slice(0, probeCellCount);
|
|
112
230
|
const stores = database.readData(root => probeCellIds.map(cellId => root.cells[cellId]));
|
|
113
231
|
if (!stores) return undefined;
|
|
114
|
-
|
|
115
|
-
const hits: SearchHit[] = [];
|
|
116
232
|
for (const store of stores) {
|
|
117
233
|
const members = replayTransactionStore<StoredEmbedding>(store);
|
|
118
|
-
for (const
|
|
119
|
-
hits.push({ ref, closeness: getCloseness(query,
|
|
234
|
+
for (const ref of members.keys()) {
|
|
235
|
+
hits.push({ ref, closeness: getCloseness(query, members.get(ref)!) });
|
|
120
236
|
}
|
|
121
237
|
}
|
|
122
238
|
hits.sort((left, right) => right.closeness - left.closeness);
|
|
123
239
|
return hits.slice(0, options.resultCount);
|
|
124
240
|
}
|
|
125
241
|
|
|
126
|
-
// Assign each embedding to its nearest cell and add it. An empty index bootstraps a first cell from the
|
|
127
|
-
// inserted batch; a cell that crosses splitAt*cellTarget splits in place. undefined while not synced.
|
|
128
242
|
export function insertEmbeddings(
|
|
129
243
|
database: Database<IvfEmbeddingRoot>,
|
|
130
244
|
items: EmbeddingInput[],
|
|
131
|
-
)
|
|
132
|
-
if (!items.length) return
|
|
245
|
+
) {
|
|
246
|
+
if (!items.length) return;
|
|
133
247
|
const config = database.readData(root => root.config);
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
if (!
|
|
248
|
+
const steps = database.readData(root => root.steps);
|
|
249
|
+
let count = database.readData(root => root.count);
|
|
250
|
+
if (!config) return;
|
|
251
|
+
if (!steps) return;
|
|
252
|
+
if (count === undefined) return;
|
|
253
|
+
const newCount = count + items.length;
|
|
137
254
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
255
|
+
if (!steps[STEP_IVF]) {
|
|
256
|
+
const flatWrites = items.map(item => ({ key: item.ref, value: item.embedding }));
|
|
257
|
+
transactionMutate(flatStore(database), flatWrites);
|
|
258
|
+
database.writeData(root => root.count, newCount);
|
|
259
|
+
if (newCount > FLAT_LIMIT) {
|
|
260
|
+
rebuildStructure(database);
|
|
261
|
+
}
|
|
262
|
+
return;
|
|
144
263
|
}
|
|
145
264
|
|
|
265
|
+
const centroids = transactionRead(centroidStore(database));
|
|
266
|
+
if (!centroids) return undefined;
|
|
146
267
|
const itemsByCell = new Map<string, EmbeddingInput[]>();
|
|
147
268
|
for (const item of items) {
|
|
148
|
-
const cellId =
|
|
269
|
+
const cellId = rankCellsByCloseness(item.embedding, centroids)[0];
|
|
149
270
|
let group = itemsByCell.get(cellId);
|
|
150
271
|
if (!group) {
|
|
151
272
|
group = [];
|
|
@@ -153,75 +274,99 @@ export function insertEmbeddings(
|
|
|
153
274
|
}
|
|
154
275
|
group.push(item);
|
|
155
276
|
}
|
|
156
|
-
|
|
157
|
-
const targetCellIds = [...itemsByCell.keys()];
|
|
158
|
-
const existingStores = database.readData(root => targetCellIds.map(cellId => root.cells[cellId]));
|
|
159
|
-
if (!existingStores) return undefined;
|
|
160
|
-
|
|
161
|
-
const splitThreshold = config.splitAt * config.cellTarget;
|
|
162
|
-
const refWrites: { key: string; value: string | undefined }[] = [];
|
|
163
|
-
for (let cellIndex = 0; cellIndex < targetCellIds.length; cellIndex++) {
|
|
164
|
-
const cellId = targetCellIds[cellIndex];
|
|
277
|
+
for (const cellId of itemsByCell.keys()) {
|
|
165
278
|
const group = itemsByCell.get(cellId)!;
|
|
166
|
-
const
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
279
|
+
const memberWrites = group.map(item => ({ key: item.ref, value: item.embedding }));
|
|
280
|
+
transactionMutate(cellStore(database, cellId), memberWrites);
|
|
281
|
+
}
|
|
282
|
+
database.writeData(root => root.count, newCount);
|
|
170
283
|
|
|
171
|
-
if (combined.size <= splitThreshold) {
|
|
172
|
-
const memberWrites = group.map(item => ({ key: item.ref, value: item.embedding }));
|
|
173
|
-
if (!transactionMutate(cellStore(database, cellId), memberWrites)) return undefined;
|
|
174
|
-
for (const item of group) {
|
|
175
|
-
refWrites.push({ key: item.ref, value: cellId });
|
|
176
|
-
}
|
|
177
|
-
continue;
|
|
178
|
-
}
|
|
179
284
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
const
|
|
185
|
-
if (!
|
|
186
|
-
|
|
187
|
-
|
|
285
|
+
const shouldRegenerate = (database: Database<IvfEmbeddingRoot>) => {
|
|
286
|
+
let steps = database.readData(root => root.steps);
|
|
287
|
+
if (!steps) return;
|
|
288
|
+
for (const threshold of REGENERATE_AT) {
|
|
289
|
+
const stepName = "regen" + threshold;
|
|
290
|
+
if (!steps[stepName] && newCount > threshold) {
|
|
291
|
+
database.writeData(root => root.steps[stepName], true);
|
|
292
|
+
return true;
|
|
188
293
|
}
|
|
189
294
|
}
|
|
190
|
-
|
|
191
|
-
|
|
295
|
+
const averageFill = newCount / Math.max(1, centroids.size) / config.cellTargetSize;
|
|
296
|
+
if (Math.random() < rebalanceProbability(averageFill)) {
|
|
297
|
+
rebuildStructure(database);
|
|
298
|
+
}
|
|
299
|
+
return true;
|
|
300
|
+
};
|
|
301
|
+
if (shouldRegenerate(database)) {
|
|
302
|
+
rebuildStructure(database);
|
|
192
303
|
}
|
|
193
|
-
|
|
194
|
-
if (newCentroids.length && !transactionMutate(centroidStore(database), newCentroids)) return undefined;
|
|
195
|
-
if (refWrites.length && !transactionMutate(refIndexStore(database), refWrites)) return undefined;
|
|
196
|
-
return true;
|
|
197
304
|
}
|
|
198
305
|
|
|
199
|
-
// Tombstone each ref in its cell (found via refIndex) and drop it from the index. undefined while not synced.
|
|
200
306
|
export function removeEmbeddings(
|
|
201
307
|
database: Database<IvfEmbeddingRoot>,
|
|
202
|
-
|
|
203
|
-
)
|
|
204
|
-
if (!
|
|
205
|
-
const
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
308
|
+
items: EmbeddingInput[],
|
|
309
|
+
) {
|
|
310
|
+
if (!items.length) return;
|
|
311
|
+
const steps = database.readData(root => root.steps);
|
|
312
|
+
const count = database.readData(root => root.count);
|
|
313
|
+
if (!steps) return;
|
|
314
|
+
if (count === undefined) return;
|
|
315
|
+
|
|
316
|
+
if (!steps[STEP_IVF]) {
|
|
317
|
+
const flatDeletes = items.map(item => ({ key: item.ref, value: undefined }));
|
|
318
|
+
transactionMutate(flatStore(database), flatDeletes);
|
|
319
|
+
database.writeData(root => root.count, Math.max(0, count - items.length));
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
const centroids = transactionRead(centroidStore(database));
|
|
324
|
+
if (!centroids) return;
|
|
325
|
+
if (!centroids.size) return;
|
|
326
|
+
|
|
327
|
+
const candidatesByItem: { ref: string; cellIds: string[] }[] = [];
|
|
328
|
+
const candidateSet = new Set<string>();
|
|
329
|
+
for (const item of items) {
|
|
330
|
+
const cellIds = rankCellsByCloseness(item.embedding, centroids).slice(0, 1 + DELETE_FALLBACK_CELLS);
|
|
331
|
+
candidatesByItem.push({ ref: item.ref, cellIds });
|
|
332
|
+
for (const cellId of cellIds) {
|
|
333
|
+
candidateSet.add(cellId);
|
|
216
334
|
}
|
|
217
|
-
group.push(ref);
|
|
218
335
|
}
|
|
219
336
|
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
337
|
+
const candidateCellIds = Array.from(candidateSet);
|
|
338
|
+
const stores = database.readData(root => candidateCellIds.map(cellId => root.cells[cellId]));
|
|
339
|
+
if (!stores) return;
|
|
340
|
+
const membersByCell = new Map<string, Map<string, StoredEmbedding>>();
|
|
341
|
+
for (let index = 0; index < candidateCellIds.length; index++) {
|
|
342
|
+
membersByCell.set(candidateCellIds[index], replayTransactionStore(stores[index]));
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
const deletesByCell = new Map<string, string[]>();
|
|
346
|
+
for (const candidate of candidatesByItem) {
|
|
347
|
+
for (const cellId of candidate.cellIds) {
|
|
348
|
+
const members = membersByCell.get(cellId);
|
|
349
|
+
if (!members || !members.has(candidate.ref)) {
|
|
350
|
+
continue;
|
|
351
|
+
}
|
|
352
|
+
let refs = deletesByCell.get(cellId);
|
|
353
|
+
if (!refs) {
|
|
354
|
+
refs = [];
|
|
355
|
+
deletesByCell.set(cellId, refs);
|
|
356
|
+
}
|
|
357
|
+
refs.push(candidate.ref);
|
|
358
|
+
break;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
let deletedCount = 0;
|
|
363
|
+
for (const cellId of deletesByCell.keys()) {
|
|
364
|
+
const refs = deletesByCell.get(cellId)!;
|
|
365
|
+
deletedCount += refs.length;
|
|
366
|
+
const memberDeletes = refs.map(ref => ({ key: ref, value: undefined }));
|
|
367
|
+
transactionMutate(cellStore(database, cellId), memberDeletes);
|
|
368
|
+
}
|
|
369
|
+
if (deletedCount) {
|
|
370
|
+
database.writeData(root => root.count, Math.max(0, count - deletedCount));
|
|
223
371
|
}
|
|
224
|
-
const refDeletes = refs.map(ref => ({ key: ref, value: undefined }));
|
|
225
|
-
if (!transactionMutate<string>(refIndexStore(database), refDeletes)) return undefined;
|
|
226
|
-
return true;
|
|
227
372
|
}
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import { Database } from "./Database";
|
|
2
|
-
|
|
2
|
+
declare const valueTag: unique symbol;
|
|
3
|
+
export type TransactionSetStore<Value> = {
|
|
3
4
|
[fileNumber: string]: Uint8Array;
|
|
5
|
+
[valueTag]?: Value;
|
|
4
6
|
};
|
|
5
|
-
export declare function transactionRead<Value>(database: Database<TransactionSetStore
|
|
6
|
-
export declare function replayTransactionStore<Value>(store: TransactionSetStore | undefined): Map<string, Value>;
|
|
7
|
-
export declare function transactionMutate<Value>(database: Database<TransactionSetStore
|
|
7
|
+
export declare function transactionRead<Value>(database: Database<TransactionSetStore<Value>>): Map<string, Value> | undefined;
|
|
8
|
+
export declare function replayTransactionStore<Value>(store: TransactionSetStore<Value> | undefined): Map<string, Value>;
|
|
9
|
+
export declare function transactionMutate<Value>(database: Database<TransactionSetStore<Value>>, transactions: {
|
|
8
10
|
key: string;
|
|
9
11
|
value: Value | undefined;
|
|
10
|
-
}[], compactAfterFiles?: number):
|
|
12
|
+
}[], compactAfterFiles?: number): void;
|
|
13
|
+
export declare function transactionDelete<Value>(database: Database<TransactionSetStore<Value>>): void;
|
|
14
|
+
export {};
|