taladb 0.11.3 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.browser.mjs +117 -22
- package/dist/index.d.mts +92 -6
- package/dist/index.d.ts +92 -6
- package/dist/index.js +118 -22
- package/dist/index.mjs +117 -22
- package/dist/index.react-native.mjs +117 -22
- package/package.json +3 -3
package/dist/index.browser.mjs
CHANGED
|
@@ -1,3 +1,89 @@
|
|
|
1
|
+
// src/vector-client.ts
|
|
2
|
+
function integer(value, name, min = 0, max = 4294967295) {
|
|
3
|
+
if (!Number.isSafeInteger(value) || value < min || value > max) throw new Error(`${name} must be an integer between ${min} and ${max}`);
|
|
4
|
+
}
|
|
5
|
+
function vectorValues(vector) {
|
|
6
|
+
const values = Array.from(vector);
|
|
7
|
+
if (values.length === 0 || values.some((v) => !Number.isFinite(v))) throw new Error("vector must contain finite numbers");
|
|
8
|
+
return values;
|
|
9
|
+
}
|
|
10
|
+
function queryOptions(options) {
|
|
11
|
+
if (options.efSearch !== void 0) integer(options.efSearch, "efSearch", 1);
|
|
12
|
+
if (options.offset !== void 0) integer(options.offset, "offset");
|
|
13
|
+
if (options.groupSize !== void 0) integer(options.groupSize, "groupSize", 1);
|
|
14
|
+
if (options.oversampling !== void 0) integer(options.oversampling, "oversampling", 1, 100);
|
|
15
|
+
if (options.scoreThreshold !== void 0 && !Number.isFinite(options.scoreThreshold)) throw new Error("scoreThreshold must be finite");
|
|
16
|
+
return options;
|
|
17
|
+
}
|
|
18
|
+
function abortError() {
|
|
19
|
+
const error = new Error("Vector index rebuild cancelled");
|
|
20
|
+
error.name = "AbortError";
|
|
21
|
+
return error;
|
|
22
|
+
}
|
|
23
|
+
function createVectorClient(send) {
|
|
24
|
+
const client = {
|
|
25
|
+
searchVectors: async (field, vector, topK, filter, options = {}) => {
|
|
26
|
+
integer(topK, "topK");
|
|
27
|
+
return send({ op: "search", field, query: vectorValues(vector), topK, filter, options: queryOptions(options) });
|
|
28
|
+
},
|
|
29
|
+
findWithin: (field, vector, scoreThreshold, filter) => client.searchVectors(field, vector, 4294967295, filter, { mode: "exact", scoreThreshold }),
|
|
30
|
+
vectorIndexStatus: (field) => send({ op: "status", field }),
|
|
31
|
+
beginVectorBuild: (field, options) => send({ op: "beginBuild", field, options }),
|
|
32
|
+
stepVectorBuild: async (field, id, batchSize = 32) => {
|
|
33
|
+
integer(batchSize, "batchSize", 1, 1024);
|
|
34
|
+
return send({ op: "stepBuild", field, id, batchSize });
|
|
35
|
+
},
|
|
36
|
+
cancelVectorBuild: (field, id) => send({ op: "cancelBuild", field, id }),
|
|
37
|
+
rebuildVectorIndex: async (field, options = {}) => {
|
|
38
|
+
const { signal, onProgress, batchSize = 32, ...graphOptions } = options;
|
|
39
|
+
integer(batchSize, "batchSize", 1, 1024);
|
|
40
|
+
if (signal?.aborted) throw abortError();
|
|
41
|
+
let progress = await client.beginVectorBuild(field, Object.keys(graphOptions).length ? graphOptions : void 0);
|
|
42
|
+
try {
|
|
43
|
+
onProgress?.(progress);
|
|
44
|
+
while (progress.state === "building") {
|
|
45
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
46
|
+
if (signal?.aborted) throw abortError();
|
|
47
|
+
progress = await client.stepVectorBuild(field, progress.id, batchSize);
|
|
48
|
+
onProgress?.(progress);
|
|
49
|
+
}
|
|
50
|
+
if (progress.state === "failed") throw new Error(progress.error ?? "Vector rebuild failed");
|
|
51
|
+
if (progress.state === "cancelled") throw abortError();
|
|
52
|
+
return progress;
|
|
53
|
+
} catch (error) {
|
|
54
|
+
if (progress.state === "building") await client.cancelVectorBuild(field, progress.id);
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
measureVectorRecall: async (field, queries, topK, filter, options = {}) => {
|
|
59
|
+
integer(topK, "topK", 1);
|
|
60
|
+
return send({ op: "recall", field, queries: queries.map(vectorValues), topK, filter, options: queryOptions(options) });
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
return client;
|
|
64
|
+
}
|
|
65
|
+
function vectorIndexRequest(field, options) {
|
|
66
|
+
integer(options.dimensions, "dimensions", 1);
|
|
67
|
+
if (options.metric && !["cosine", "dot", "euclidean"].includes(options.metric)) throw new Error("invalid vector metric");
|
|
68
|
+
if (options.indexType && !["flat", "hnsw"].includes(options.indexType)) throw new Error("invalid vector indexType");
|
|
69
|
+
if (options.quantization && options.quantization !== "none" && options.indexType !== "hnsw") throw new Error("quantization requires an HNSW index");
|
|
70
|
+
if (options.indexType === "hnsw") {
|
|
71
|
+
const m = options.hnswM ?? 32;
|
|
72
|
+
const efConstruction = options.hnswEfConstruction ?? 200;
|
|
73
|
+
integer(m, "hnswM", 2, 128);
|
|
74
|
+
integer(efConstruction, "hnswEfConstruction", m, 1e5);
|
|
75
|
+
if (options.metric === "dot") throw new Error("HNSW requires cosine or euclidean");
|
|
76
|
+
if (options.quantization === "binary" && options.metric && options.metric !== "cosine") throw new Error("binary quantization requires cosine");
|
|
77
|
+
}
|
|
78
|
+
return {
|
|
79
|
+
op: "create",
|
|
80
|
+
field,
|
|
81
|
+
dimensions: options.dimensions,
|
|
82
|
+
metric: options.metric,
|
|
83
|
+
options: options.indexType === "hnsw" ? { m: options.hnswM ?? 32, efConstruction: options.hnswEfConstruction ?? 200, quantization: options.quantization ?? "none" } : null
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
1
87
|
// src/config.browser.ts
|
|
2
88
|
var ENDPOINT_FIELDS = [
|
|
3
89
|
"endpoint",
|
|
@@ -710,7 +796,9 @@ async function createBrowserDB(dbName, webhook, config, passphrase, migrations)
|
|
|
710
796
|
}
|
|
711
797
|
function wrapCollection(name, opts) {
|
|
712
798
|
const s = JSON.stringify;
|
|
799
|
+
const vectorCommand = async (request) => JSON.parse(await proxy.send("vectorCommand", { collection: name, requestJson: JSON.stringify(request) }));
|
|
713
800
|
const wrapped = {
|
|
801
|
+
...createVectorClient(vectorCommand),
|
|
714
802
|
insert: (doc) => proxy.send("insert", { collection: name, docJson: s(doc) }),
|
|
715
803
|
insertMany: async (docs) => {
|
|
716
804
|
const json = await proxy.send("insertMany", {
|
|
@@ -762,25 +850,21 @@ async function createBrowserDB(dbName, webhook, config, passphrase, migrations)
|
|
|
762
850
|
dropCompoundIndex: (fields) => proxy.send("dropCompoundIndex", { collection: name, fieldsJson: JSON.stringify(fields) }),
|
|
763
851
|
createFtsIndex: (field) => proxy.send("createFtsIndex", { collection: name, field }),
|
|
764
852
|
dropFtsIndex: (field) => proxy.send("dropFtsIndex", { collection: name, field }),
|
|
765
|
-
createVectorIndex: (field, options) => {
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
field,
|
|
770
|
-
dimensions: options.dimensions,
|
|
771
|
-
metric: options.metric,
|
|
772
|
-
indexType: null,
|
|
773
|
-
hnswM: null,
|
|
774
|
-
hnswEfConstruction: null
|
|
775
|
-
});
|
|
853
|
+
createVectorIndex: async (field, options) => {
|
|
854
|
+
const request = vectorIndexRequest(field, options);
|
|
855
|
+
await vectorCommand({ ...request, deferBuild: true });
|
|
856
|
+
if (request.options) await createVectorClient(vectorCommand).rebuildVectorIndex(field, request.options);
|
|
776
857
|
},
|
|
777
858
|
dropVectorIndex: (field) => proxy.send("dropVectorIndex", { collection: name, field }),
|
|
778
|
-
upgradeVectorIndex: (
|
|
859
|
+
upgradeVectorIndex: async (field) => {
|
|
860
|
+
await createVectorClient(vectorCommand).rebuildVectorIndex(field);
|
|
861
|
+
},
|
|
779
862
|
listIndexes: async () => {
|
|
780
863
|
const json = await proxy.send("listIndexes", { collection: name });
|
|
781
864
|
return JSON.parse(json);
|
|
782
865
|
},
|
|
783
|
-
findNearest: async (field, vector, topK, filter) => {
|
|
866
|
+
findNearest: async (field, vector, topK, filter, options) => {
|
|
867
|
+
if (options) return (await createVectorClient(vectorCommand).searchVectors(field, vector, topK, filter, options)).hits;
|
|
784
868
|
const json = await proxy.send("findNearest", {
|
|
785
869
|
collection: name,
|
|
786
870
|
field,
|
|
@@ -941,7 +1025,9 @@ async function createNodeDB(dbName, webhook, config, passphrase, migrations) {
|
|
|
941
1025
|
const db = TalaDBNode.open(dbName, configJson, passphrase ?? null);
|
|
942
1026
|
function wrapCollection(name, opts) {
|
|
943
1027
|
const col = db.collection(name);
|
|
1028
|
+
const vectorCommand = async (request) => col.vectorCommandAsync ? col.vectorCommandAsync(request) : col.vectorCommand(request);
|
|
944
1029
|
const wrapped = {
|
|
1030
|
+
...createVectorClient(vectorCommand),
|
|
945
1031
|
insert: async (doc) => col.insertAsync ? col.insertAsync(doc) : col.insert(doc),
|
|
946
1032
|
insertMany: async (docs) => col.insertManyAsync ? col.insertManyAsync(docs) : col.insertMany(docs),
|
|
947
1033
|
find: async (filter) => col.findAsync ? col.findAsync(filter ?? null) : col.find(filter ?? null),
|
|
@@ -958,14 +1044,19 @@ async function createNodeDB(dbName, webhook, config, passphrase, migrations) {
|
|
|
958
1044
|
dropCompoundIndex: async (fields) => col.dropCompoundIndex(fields),
|
|
959
1045
|
createFtsIndex: async (field) => col.createFtsIndex(field),
|
|
960
1046
|
dropFtsIndex: async (field) => col.dropFtsIndex(field),
|
|
961
|
-
createVectorIndex: async (field, options) =>
|
|
1047
|
+
createVectorIndex: async (field, options) => {
|
|
1048
|
+
const request = vectorIndexRequest(field, options);
|
|
1049
|
+
await vectorCommand({ ...request, deferBuild: true });
|
|
1050
|
+
if (request.options) await createVectorClient(vectorCommand).rebuildVectorIndex(field, request.options);
|
|
1051
|
+
},
|
|
962
1052
|
dropVectorIndex: async (field) => col.dropVectorIndex(field),
|
|
963
1053
|
upgradeVectorIndex: async (field) => col.upgradeVectorIndex(field),
|
|
964
1054
|
listIndexes: async () => {
|
|
965
1055
|
const json = col.listIndexes();
|
|
966
1056
|
return JSON.parse(json);
|
|
967
1057
|
},
|
|
968
|
-
findNearest: async (field, vector, topK, filter) => {
|
|
1058
|
+
findNearest: async (field, vector, topK, filter, options) => {
|
|
1059
|
+
if (options) return (await createVectorClient(vectorCommand).searchVectors(field, vector, topK, filter, options)).hits;
|
|
969
1060
|
const raw = await col.findNearest(field, vector, topK, filter ?? null);
|
|
970
1061
|
return raw;
|
|
971
1062
|
},
|
|
@@ -1044,7 +1135,9 @@ async function createNativeDB(_dbName, webhook, migrations) {
|
|
|
1044
1135
|
return result;
|
|
1045
1136
|
};
|
|
1046
1137
|
function wrapCollection(name, opts) {
|
|
1138
|
+
const vectorCommand = async (request) => call("vectorCommand", name, request);
|
|
1047
1139
|
const wrapped = {
|
|
1140
|
+
...createVectorClient(vectorCommand),
|
|
1048
1141
|
insert: async (doc) => await call("insert", name, doc),
|
|
1049
1142
|
insertMany: async (docs) => await call("insertMany", name, docs),
|
|
1050
1143
|
find: async (filter) => await call("find", name, filter ?? {}),
|
|
@@ -1062,17 +1155,15 @@ async function createNativeDB(_dbName, webhook, migrations) {
|
|
|
1062
1155
|
createFtsIndex: async (field) => await call("createFtsIndex", name, field),
|
|
1063
1156
|
dropFtsIndex: async (field) => await call("dropFtsIndex", name, field),
|
|
1064
1157
|
createVectorIndex: async (field, options) => {
|
|
1065
|
-
const
|
|
1066
|
-
|
|
1067
|
-
if (options
|
|
1068
|
-
opts2.hnsw = { m: options.hnswM ?? 32, ef_construction: options.hnswEfConstruction ?? 200 };
|
|
1069
|
-
}
|
|
1070
|
-
return await call("createVectorIndex", name, field, options.dimensions, opts2);
|
|
1158
|
+
const request = vectorIndexRequest(field, options);
|
|
1159
|
+
await vectorCommand({ ...request, deferBuild: true });
|
|
1160
|
+
if (request.options) await createVectorClient(vectorCommand).rebuildVectorIndex(field, request.options);
|
|
1071
1161
|
},
|
|
1072
1162
|
dropVectorIndex: async (field) => await call("dropVectorIndex", name, field),
|
|
1073
1163
|
upgradeVectorIndex: async (field) => await call("upgradeVectorIndex", name, field),
|
|
1074
1164
|
listIndexes: async () => call("listIndexes", name),
|
|
1075
|
-
findNearest: async (field, vector, topK, filter) => {
|
|
1165
|
+
findNearest: async (field, vector, topK, filter, options) => {
|
|
1166
|
+
if (options) return (await createVectorClient(vectorCommand).searchVectors(field, vector, topK, filter, options)).hits;
|
|
1076
1167
|
const raw = await call("findNearest", name, field, vector, topK, filter ?? null);
|
|
1077
1168
|
return raw;
|
|
1078
1169
|
},
|
|
@@ -1100,6 +1191,9 @@ async function createNativeDB(_dbName, webhook, migrations) {
|
|
|
1100
1191
|
flush: native.callAsync || native.flush ? async () => {
|
|
1101
1192
|
await call("flush");
|
|
1102
1193
|
} : void 0,
|
|
1194
|
+
rebuildVectorIndexes: native.callAsync || native.rebuildVectorIndexes ? async () => {
|
|
1195
|
+
await call("rebuildVectorIndexes");
|
|
1196
|
+
} : void 0,
|
|
1103
1197
|
// One process owns the file — there is no other tab to defer to.
|
|
1104
1198
|
isPrimary: async () => true
|
|
1105
1199
|
};
|
|
@@ -1187,6 +1281,7 @@ function attachWebhook(db, webhook) {
|
|
|
1187
1281
|
export {
|
|
1188
1282
|
TalaDbValidationError,
|
|
1189
1283
|
applySchema,
|
|
1284
|
+
createVectorClient,
|
|
1190
1285
|
createWebhookDispatcher,
|
|
1191
1286
|
decorateCollection,
|
|
1192
1287
|
deriveDocId,
|
package/dist/index.d.mts
CHANGED
|
@@ -1,3 +1,81 @@
|
|
|
1
|
+
/** Shared vector API for Node, browser workers, and the React Native job executor. */
|
|
2
|
+
type VectorQuantization = 'none' | 'scalar' | 'binary';
|
|
3
|
+
interface VectorGraphOptions {
|
|
4
|
+
m?: number;
|
|
5
|
+
efConstruction?: number;
|
|
6
|
+
quantization?: VectorQuantization;
|
|
7
|
+
}
|
|
8
|
+
interface VectorQueryOptions {
|
|
9
|
+
mode?: 'auto' | 'exact' | 'ann';
|
|
10
|
+
efSearch?: number;
|
|
11
|
+
scoreThreshold?: number;
|
|
12
|
+
offset?: number;
|
|
13
|
+
groupBy?: string;
|
|
14
|
+
groupSize?: number;
|
|
15
|
+
oversampling?: number;
|
|
16
|
+
}
|
|
17
|
+
interface VectorBuildProgress {
|
|
18
|
+
id: string;
|
|
19
|
+
state: 'building' | 'ready' | 'cancelled' | 'failed';
|
|
20
|
+
processed: number;
|
|
21
|
+
total: number;
|
|
22
|
+
revision: number;
|
|
23
|
+
error: string | null;
|
|
24
|
+
}
|
|
25
|
+
interface VectorRebuildOptions extends VectorGraphOptions {
|
|
26
|
+
/** Insertions per native/worker call (1–1024); defaults to 32 for mobile responsiveness. */
|
|
27
|
+
batchSize?: number;
|
|
28
|
+
signal?: AbortSignal;
|
|
29
|
+
onProgress?: (progress: VectorBuildProgress) => void;
|
|
30
|
+
}
|
|
31
|
+
interface VectorIndexStatus {
|
|
32
|
+
field: string;
|
|
33
|
+
state: 'flat' | 'ready' | 'stale' | 'rebuildRequired';
|
|
34
|
+
persistent: boolean;
|
|
35
|
+
indexedVectors: number;
|
|
36
|
+
totalVectors: number;
|
|
37
|
+
deletedNodes: number;
|
|
38
|
+
revision: number;
|
|
39
|
+
indexRevision: number | null;
|
|
40
|
+
options: Required<VectorGraphOptions> | null;
|
|
41
|
+
build: VectorBuildProgress | null;
|
|
42
|
+
}
|
|
43
|
+
interface VectorQueryResult<T> {
|
|
44
|
+
hits: {
|
|
45
|
+
document: T;
|
|
46
|
+
score: number;
|
|
47
|
+
}[];
|
|
48
|
+
execution: {
|
|
49
|
+
path: 'exact' | 'hnsw';
|
|
50
|
+
reason: string;
|
|
51
|
+
revision: number;
|
|
52
|
+
efSearch: number | null;
|
|
53
|
+
distanceComputations: number;
|
|
54
|
+
};
|
|
55
|
+
/** Offset pagination uses live snapshots. Writes between pages can change ordering. */
|
|
56
|
+
nextOffset: number | null;
|
|
57
|
+
}
|
|
58
|
+
interface VectorRecall {
|
|
59
|
+
recallAtK: number;
|
|
60
|
+
queries: number;
|
|
61
|
+
topK: number;
|
|
62
|
+
exactMs: number;
|
|
63
|
+
annMs: number;
|
|
64
|
+
}
|
|
65
|
+
interface VectorClient<T> {
|
|
66
|
+
searchVectors(field: string, vector: ArrayLike<number>, topK: number, filter?: Record<string, unknown>, options?: VectorQueryOptions): Promise<VectorQueryResult<T>>;
|
|
67
|
+
/** Exact range search. Returns every matching document above the score threshold. */
|
|
68
|
+
findWithin(field: string, vector: ArrayLike<number>, scoreThreshold: number, filter?: Record<string, unknown>): Promise<VectorQueryResult<T>>;
|
|
69
|
+
vectorIndexStatus(field: string): Promise<VectorIndexStatus>;
|
|
70
|
+
rebuildVectorIndex(field: string, options?: VectorRebuildOptions): Promise<VectorBuildProgress>;
|
|
71
|
+
beginVectorBuild(field: string, options?: VectorGraphOptions): Promise<VectorBuildProgress>;
|
|
72
|
+
stepVectorBuild(field: string, id: string, batchSize?: number): Promise<VectorBuildProgress>;
|
|
73
|
+
cancelVectorBuild(field: string, id: string): Promise<VectorBuildProgress>;
|
|
74
|
+
measureVectorRecall(field: string, queries: ArrayLike<number>[], topK: number, filter?: Record<string, unknown>, options?: VectorQueryOptions): Promise<VectorRecall>;
|
|
75
|
+
}
|
|
76
|
+
/** Binding adapter used by TalaDB's Node, browser, and React Native packages. */
|
|
77
|
+
declare function createVectorClient<T>(send: (request: Record<string, unknown>) => Promise<any>): VectorClient<T>;
|
|
78
|
+
|
|
1
79
|
/** The three mutation kinds a webhook reports, and the verb each one uses. */
|
|
2
80
|
type WebhookOp = 'insert' | 'update' | 'delete';
|
|
3
81
|
interface WebhookConfig {
|
|
@@ -105,13 +183,15 @@ interface VectorIndexOptions {
|
|
|
105
183
|
* Index algorithm. Defaults to `"flat"` (exact brute-force).
|
|
106
184
|
* Use `"hnsw"` for approximate nearest-neighbour search — much faster on
|
|
107
185
|
* large collections at the cost of occasional missed results.
|
|
108
|
-
*
|
|
186
|
+
* Supported on browser, Node.js and React Native.
|
|
109
187
|
*/
|
|
110
188
|
indexType?: 'flat' | 'hnsw';
|
|
111
|
-
/** HNSW connectivity parameter M.
|
|
189
|
+
/** HNSW connectivity parameter M. Supported range is 2–128; defaults to 32. */
|
|
112
190
|
hnswM?: number;
|
|
113
191
|
/** HNSW build-time quality parameter ef_construction (default 200). */
|
|
114
192
|
hnswEfConstruction?: number;
|
|
193
|
+
/** Graph compression; originals remain available for exact rescoring. */
|
|
194
|
+
quantization?: VectorQuantization;
|
|
115
195
|
}
|
|
116
196
|
/** Describes the indexes that exist on a collection. */
|
|
117
197
|
interface CollectionIndexInfo {
|
|
@@ -482,7 +562,7 @@ type AggregatePipeline<T extends Document = Document> = AggregateStage<T>[];
|
|
|
482
562
|
type InsertDoc<T extends Document> = Omit<T, '_id'> & {
|
|
483
563
|
_id?: string;
|
|
484
564
|
};
|
|
485
|
-
interface Collection<T extends Document = Document> {
|
|
565
|
+
interface Collection<T extends Document = Document> extends VectorClient<T> {
|
|
486
566
|
insert(doc: InsertDoc<T>): Promise<string>;
|
|
487
567
|
insertMany(docs: InsertDoc<T>[]): Promise<string[]>;
|
|
488
568
|
find(filter?: Filter<T>): Promise<T[]>;
|
|
@@ -600,7 +680,7 @@ interface Collection<T extends Document = Document> {
|
|
|
600
680
|
*
|
|
601
681
|
* After calling this, `findNearest` uses approximate nearest-neighbour
|
|
602
682
|
* search which is significantly faster on large collections.
|
|
603
|
-
*
|
|
683
|
+
* Promotes flat/legacy indexes and compacts persistent graphs.
|
|
604
684
|
*/
|
|
605
685
|
upgradeVectorIndex(field: keyof Omit<T, '_id'> & string): Promise<void>;
|
|
606
686
|
/**
|
|
@@ -620,7 +700,7 @@ interface Collection<T extends Document = Document> {
|
|
|
620
700
|
* locale: 'en',
|
|
621
701
|
* });
|
|
622
702
|
*/
|
|
623
|
-
findNearest(field: keyof Omit<T, '_id'> & string, vector: number[], topK: number, filter?: Filter<T
|
|
703
|
+
findNearest(field: keyof Omit<T, '_id'> & string, vector: number[], topK: number, filter?: Filter<T>, options?: VectorQueryOptions): Promise<VectorSearchResult<T>[]>;
|
|
624
704
|
/**
|
|
625
705
|
* Subscribe to live query results. The callback receives a full snapshot of
|
|
626
706
|
* matching documents immediately and again after every write that could
|
|
@@ -680,6 +760,12 @@ interface TalaDB {
|
|
|
680
760
|
* "save now" moments (before checkout, on `visibilitychange`).
|
|
681
761
|
*/
|
|
682
762
|
flush?(): Promise<void>;
|
|
763
|
+
/**
|
|
764
|
+
* Rebuild all configured HNSW graphs for maintenance. Graphs persist across
|
|
765
|
+
* restarts; startup rebuilding is unnecessary. Prefer per-collection
|
|
766
|
+
* rebuildVectorIndex for bounded batches, progress and cancellation.
|
|
767
|
+
*/
|
|
768
|
+
rebuildVectorIndexes?(): Promise<void>;
|
|
683
769
|
/**
|
|
684
770
|
* Whether this browser tab owns the database storage. All tabs execute reads
|
|
685
771
|
* and writes through that owner and await its result. Ownership can change
|
|
@@ -932,4 +1018,4 @@ interface OpenDBOptions {
|
|
|
932
1018
|
*/
|
|
933
1019
|
declare function openDB(dbName?: string, options?: OpenDBOptions): Promise<TalaDB>;
|
|
934
1020
|
|
|
935
|
-
export { type AggregatePipeline, type AggregateStage, type Collection, type CollectionIndexInfo, type CollectionOptions, type Document, type DurabilityConfig, type Filter, type HybridSearchOptions, type HybridSearchResult, type InsertDoc, type Migration, type OpenDBOptions, type Schema, type TalaDB, type TalaDbConfig, TalaDbValidationError, type TextSearchOptions, type TextSearchResult, type Update, type Value, type VectorIndexOptions, type VectorMetric, type VectorSearchResult, type WebhookConfig, type WebhookDispatcher, type WebhookEvent, type WebhookOp, type WebhookStats, applySchema, createWebhookDispatcher, decorateCollection, deriveDocId, isDocId, openDB, runMigrations, validateWebhookConfig };
|
|
1021
|
+
export { type AggregatePipeline, type AggregateStage, type Collection, type CollectionIndexInfo, type CollectionOptions, type Document, type DurabilityConfig, type Filter, type HybridSearchOptions, type HybridSearchResult, type InsertDoc, type Migration, type OpenDBOptions, type Schema, type TalaDB, type TalaDbConfig, TalaDbValidationError, type TextSearchOptions, type TextSearchResult, type Update, type Value, type VectorBuildProgress, type VectorClient, type VectorGraphOptions, type VectorIndexOptions, type VectorIndexStatus, type VectorMetric, type VectorQuantization, type VectorQueryOptions, type VectorQueryResult, type VectorRebuildOptions, type VectorRecall, type VectorSearchResult, type WebhookConfig, type WebhookDispatcher, type WebhookEvent, type WebhookOp, type WebhookStats, applySchema, createVectorClient, createWebhookDispatcher, decorateCollection, deriveDocId, isDocId, openDB, runMigrations, validateWebhookConfig };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,81 @@
|
|
|
1
|
+
/** Shared vector API for Node, browser workers, and the React Native job executor. */
|
|
2
|
+
type VectorQuantization = 'none' | 'scalar' | 'binary';
|
|
3
|
+
interface VectorGraphOptions {
|
|
4
|
+
m?: number;
|
|
5
|
+
efConstruction?: number;
|
|
6
|
+
quantization?: VectorQuantization;
|
|
7
|
+
}
|
|
8
|
+
interface VectorQueryOptions {
|
|
9
|
+
mode?: 'auto' | 'exact' | 'ann';
|
|
10
|
+
efSearch?: number;
|
|
11
|
+
scoreThreshold?: number;
|
|
12
|
+
offset?: number;
|
|
13
|
+
groupBy?: string;
|
|
14
|
+
groupSize?: number;
|
|
15
|
+
oversampling?: number;
|
|
16
|
+
}
|
|
17
|
+
interface VectorBuildProgress {
|
|
18
|
+
id: string;
|
|
19
|
+
state: 'building' | 'ready' | 'cancelled' | 'failed';
|
|
20
|
+
processed: number;
|
|
21
|
+
total: number;
|
|
22
|
+
revision: number;
|
|
23
|
+
error: string | null;
|
|
24
|
+
}
|
|
25
|
+
interface VectorRebuildOptions extends VectorGraphOptions {
|
|
26
|
+
/** Insertions per native/worker call (1–1024); defaults to 32 for mobile responsiveness. */
|
|
27
|
+
batchSize?: number;
|
|
28
|
+
signal?: AbortSignal;
|
|
29
|
+
onProgress?: (progress: VectorBuildProgress) => void;
|
|
30
|
+
}
|
|
31
|
+
interface VectorIndexStatus {
|
|
32
|
+
field: string;
|
|
33
|
+
state: 'flat' | 'ready' | 'stale' | 'rebuildRequired';
|
|
34
|
+
persistent: boolean;
|
|
35
|
+
indexedVectors: number;
|
|
36
|
+
totalVectors: number;
|
|
37
|
+
deletedNodes: number;
|
|
38
|
+
revision: number;
|
|
39
|
+
indexRevision: number | null;
|
|
40
|
+
options: Required<VectorGraphOptions> | null;
|
|
41
|
+
build: VectorBuildProgress | null;
|
|
42
|
+
}
|
|
43
|
+
interface VectorQueryResult<T> {
|
|
44
|
+
hits: {
|
|
45
|
+
document: T;
|
|
46
|
+
score: number;
|
|
47
|
+
}[];
|
|
48
|
+
execution: {
|
|
49
|
+
path: 'exact' | 'hnsw';
|
|
50
|
+
reason: string;
|
|
51
|
+
revision: number;
|
|
52
|
+
efSearch: number | null;
|
|
53
|
+
distanceComputations: number;
|
|
54
|
+
};
|
|
55
|
+
/** Offset pagination uses live snapshots. Writes between pages can change ordering. */
|
|
56
|
+
nextOffset: number | null;
|
|
57
|
+
}
|
|
58
|
+
interface VectorRecall {
|
|
59
|
+
recallAtK: number;
|
|
60
|
+
queries: number;
|
|
61
|
+
topK: number;
|
|
62
|
+
exactMs: number;
|
|
63
|
+
annMs: number;
|
|
64
|
+
}
|
|
65
|
+
interface VectorClient<T> {
|
|
66
|
+
searchVectors(field: string, vector: ArrayLike<number>, topK: number, filter?: Record<string, unknown>, options?: VectorQueryOptions): Promise<VectorQueryResult<T>>;
|
|
67
|
+
/** Exact range search. Returns every matching document above the score threshold. */
|
|
68
|
+
findWithin(field: string, vector: ArrayLike<number>, scoreThreshold: number, filter?: Record<string, unknown>): Promise<VectorQueryResult<T>>;
|
|
69
|
+
vectorIndexStatus(field: string): Promise<VectorIndexStatus>;
|
|
70
|
+
rebuildVectorIndex(field: string, options?: VectorRebuildOptions): Promise<VectorBuildProgress>;
|
|
71
|
+
beginVectorBuild(field: string, options?: VectorGraphOptions): Promise<VectorBuildProgress>;
|
|
72
|
+
stepVectorBuild(field: string, id: string, batchSize?: number): Promise<VectorBuildProgress>;
|
|
73
|
+
cancelVectorBuild(field: string, id: string): Promise<VectorBuildProgress>;
|
|
74
|
+
measureVectorRecall(field: string, queries: ArrayLike<number>[], topK: number, filter?: Record<string, unknown>, options?: VectorQueryOptions): Promise<VectorRecall>;
|
|
75
|
+
}
|
|
76
|
+
/** Binding adapter used by TalaDB's Node, browser, and React Native packages. */
|
|
77
|
+
declare function createVectorClient<T>(send: (request: Record<string, unknown>) => Promise<any>): VectorClient<T>;
|
|
78
|
+
|
|
1
79
|
/** The three mutation kinds a webhook reports, and the verb each one uses. */
|
|
2
80
|
type WebhookOp = 'insert' | 'update' | 'delete';
|
|
3
81
|
interface WebhookConfig {
|
|
@@ -105,13 +183,15 @@ interface VectorIndexOptions {
|
|
|
105
183
|
* Index algorithm. Defaults to `"flat"` (exact brute-force).
|
|
106
184
|
* Use `"hnsw"` for approximate nearest-neighbour search — much faster on
|
|
107
185
|
* large collections at the cost of occasional missed results.
|
|
108
|
-
*
|
|
186
|
+
* Supported on browser, Node.js and React Native.
|
|
109
187
|
*/
|
|
110
188
|
indexType?: 'flat' | 'hnsw';
|
|
111
|
-
/** HNSW connectivity parameter M.
|
|
189
|
+
/** HNSW connectivity parameter M. Supported range is 2–128; defaults to 32. */
|
|
112
190
|
hnswM?: number;
|
|
113
191
|
/** HNSW build-time quality parameter ef_construction (default 200). */
|
|
114
192
|
hnswEfConstruction?: number;
|
|
193
|
+
/** Graph compression; originals remain available for exact rescoring. */
|
|
194
|
+
quantization?: VectorQuantization;
|
|
115
195
|
}
|
|
116
196
|
/** Describes the indexes that exist on a collection. */
|
|
117
197
|
interface CollectionIndexInfo {
|
|
@@ -482,7 +562,7 @@ type AggregatePipeline<T extends Document = Document> = AggregateStage<T>[];
|
|
|
482
562
|
type InsertDoc<T extends Document> = Omit<T, '_id'> & {
|
|
483
563
|
_id?: string;
|
|
484
564
|
};
|
|
485
|
-
interface Collection<T extends Document = Document> {
|
|
565
|
+
interface Collection<T extends Document = Document> extends VectorClient<T> {
|
|
486
566
|
insert(doc: InsertDoc<T>): Promise<string>;
|
|
487
567
|
insertMany(docs: InsertDoc<T>[]): Promise<string[]>;
|
|
488
568
|
find(filter?: Filter<T>): Promise<T[]>;
|
|
@@ -600,7 +680,7 @@ interface Collection<T extends Document = Document> {
|
|
|
600
680
|
*
|
|
601
681
|
* After calling this, `findNearest` uses approximate nearest-neighbour
|
|
602
682
|
* search which is significantly faster on large collections.
|
|
603
|
-
*
|
|
683
|
+
* Promotes flat/legacy indexes and compacts persistent graphs.
|
|
604
684
|
*/
|
|
605
685
|
upgradeVectorIndex(field: keyof Omit<T, '_id'> & string): Promise<void>;
|
|
606
686
|
/**
|
|
@@ -620,7 +700,7 @@ interface Collection<T extends Document = Document> {
|
|
|
620
700
|
* locale: 'en',
|
|
621
701
|
* });
|
|
622
702
|
*/
|
|
623
|
-
findNearest(field: keyof Omit<T, '_id'> & string, vector: number[], topK: number, filter?: Filter<T
|
|
703
|
+
findNearest(field: keyof Omit<T, '_id'> & string, vector: number[], topK: number, filter?: Filter<T>, options?: VectorQueryOptions): Promise<VectorSearchResult<T>[]>;
|
|
624
704
|
/**
|
|
625
705
|
* Subscribe to live query results. The callback receives a full snapshot of
|
|
626
706
|
* matching documents immediately and again after every write that could
|
|
@@ -680,6 +760,12 @@ interface TalaDB {
|
|
|
680
760
|
* "save now" moments (before checkout, on `visibilitychange`).
|
|
681
761
|
*/
|
|
682
762
|
flush?(): Promise<void>;
|
|
763
|
+
/**
|
|
764
|
+
* Rebuild all configured HNSW graphs for maintenance. Graphs persist across
|
|
765
|
+
* restarts; startup rebuilding is unnecessary. Prefer per-collection
|
|
766
|
+
* rebuildVectorIndex for bounded batches, progress and cancellation.
|
|
767
|
+
*/
|
|
768
|
+
rebuildVectorIndexes?(): Promise<void>;
|
|
683
769
|
/**
|
|
684
770
|
* Whether this browser tab owns the database storage. All tabs execute reads
|
|
685
771
|
* and writes through that owner and await its result. Ownership can change
|
|
@@ -932,4 +1018,4 @@ interface OpenDBOptions {
|
|
|
932
1018
|
*/
|
|
933
1019
|
declare function openDB(dbName?: string, options?: OpenDBOptions): Promise<TalaDB>;
|
|
934
1020
|
|
|
935
|
-
export { type AggregatePipeline, type AggregateStage, type Collection, type CollectionIndexInfo, type CollectionOptions, type Document, type DurabilityConfig, type Filter, type HybridSearchOptions, type HybridSearchResult, type InsertDoc, type Migration, type OpenDBOptions, type Schema, type TalaDB, type TalaDbConfig, TalaDbValidationError, type TextSearchOptions, type TextSearchResult, type Update, type Value, type VectorIndexOptions, type VectorMetric, type VectorSearchResult, type WebhookConfig, type WebhookDispatcher, type WebhookEvent, type WebhookOp, type WebhookStats, applySchema, createWebhookDispatcher, decorateCollection, deriveDocId, isDocId, openDB, runMigrations, validateWebhookConfig };
|
|
1021
|
+
export { type AggregatePipeline, type AggregateStage, type Collection, type CollectionIndexInfo, type CollectionOptions, type Document, type DurabilityConfig, type Filter, type HybridSearchOptions, type HybridSearchResult, type InsertDoc, type Migration, type OpenDBOptions, type Schema, type TalaDB, type TalaDbConfig, TalaDbValidationError, type TextSearchOptions, type TextSearchResult, type Update, type Value, type VectorBuildProgress, type VectorClient, type VectorGraphOptions, type VectorIndexOptions, type VectorIndexStatus, type VectorMetric, type VectorQuantization, type VectorQueryOptions, type VectorQueryResult, type VectorRebuildOptions, type VectorRecall, type VectorSearchResult, type WebhookConfig, type WebhookDispatcher, type WebhookEvent, type WebhookOp, type WebhookStats, applySchema, createVectorClient, createWebhookDispatcher, decorateCollection, deriveDocId, isDocId, openDB, runMigrations, validateWebhookConfig };
|
package/dist/index.js
CHANGED
|
@@ -32,6 +32,7 @@ var index_exports = {};
|
|
|
32
32
|
__export(index_exports, {
|
|
33
33
|
TalaDbValidationError: () => TalaDbValidationError,
|
|
34
34
|
applySchema: () => applySchema,
|
|
35
|
+
createVectorClient: () => createVectorClient,
|
|
35
36
|
createWebhookDispatcher: () => createWebhookDispatcher,
|
|
36
37
|
decorateCollection: () => decorateCollection,
|
|
37
38
|
deriveDocId: () => deriveDocId,
|
|
@@ -42,6 +43,92 @@ __export(index_exports, {
|
|
|
42
43
|
});
|
|
43
44
|
module.exports = __toCommonJS(index_exports);
|
|
44
45
|
|
|
46
|
+
// src/vector-client.ts
|
|
47
|
+
function integer(value, name, min = 0, max = 4294967295) {
|
|
48
|
+
if (!Number.isSafeInteger(value) || value < min || value > max) throw new Error(`${name} must be an integer between ${min} and ${max}`);
|
|
49
|
+
}
|
|
50
|
+
function vectorValues(vector) {
|
|
51
|
+
const values = Array.from(vector);
|
|
52
|
+
if (values.length === 0 || values.some((v) => !Number.isFinite(v))) throw new Error("vector must contain finite numbers");
|
|
53
|
+
return values;
|
|
54
|
+
}
|
|
55
|
+
function queryOptions(options) {
|
|
56
|
+
if (options.efSearch !== void 0) integer(options.efSearch, "efSearch", 1);
|
|
57
|
+
if (options.offset !== void 0) integer(options.offset, "offset");
|
|
58
|
+
if (options.groupSize !== void 0) integer(options.groupSize, "groupSize", 1);
|
|
59
|
+
if (options.oversampling !== void 0) integer(options.oversampling, "oversampling", 1, 100);
|
|
60
|
+
if (options.scoreThreshold !== void 0 && !Number.isFinite(options.scoreThreshold)) throw new Error("scoreThreshold must be finite");
|
|
61
|
+
return options;
|
|
62
|
+
}
|
|
63
|
+
function abortError() {
|
|
64
|
+
const error = new Error("Vector index rebuild cancelled");
|
|
65
|
+
error.name = "AbortError";
|
|
66
|
+
return error;
|
|
67
|
+
}
|
|
68
|
+
function createVectorClient(send) {
|
|
69
|
+
const client = {
|
|
70
|
+
searchVectors: async (field, vector, topK, filter, options = {}) => {
|
|
71
|
+
integer(topK, "topK");
|
|
72
|
+
return send({ op: "search", field, query: vectorValues(vector), topK, filter, options: queryOptions(options) });
|
|
73
|
+
},
|
|
74
|
+
findWithin: (field, vector, scoreThreshold, filter) => client.searchVectors(field, vector, 4294967295, filter, { mode: "exact", scoreThreshold }),
|
|
75
|
+
vectorIndexStatus: (field) => send({ op: "status", field }),
|
|
76
|
+
beginVectorBuild: (field, options) => send({ op: "beginBuild", field, options }),
|
|
77
|
+
stepVectorBuild: async (field, id, batchSize = 32) => {
|
|
78
|
+
integer(batchSize, "batchSize", 1, 1024);
|
|
79
|
+
return send({ op: "stepBuild", field, id, batchSize });
|
|
80
|
+
},
|
|
81
|
+
cancelVectorBuild: (field, id) => send({ op: "cancelBuild", field, id }),
|
|
82
|
+
rebuildVectorIndex: async (field, options = {}) => {
|
|
83
|
+
const { signal, onProgress, batchSize = 32, ...graphOptions } = options;
|
|
84
|
+
integer(batchSize, "batchSize", 1, 1024);
|
|
85
|
+
if (signal?.aborted) throw abortError();
|
|
86
|
+
let progress = await client.beginVectorBuild(field, Object.keys(graphOptions).length ? graphOptions : void 0);
|
|
87
|
+
try {
|
|
88
|
+
onProgress?.(progress);
|
|
89
|
+
while (progress.state === "building") {
|
|
90
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
91
|
+
if (signal?.aborted) throw abortError();
|
|
92
|
+
progress = await client.stepVectorBuild(field, progress.id, batchSize);
|
|
93
|
+
onProgress?.(progress);
|
|
94
|
+
}
|
|
95
|
+
if (progress.state === "failed") throw new Error(progress.error ?? "Vector rebuild failed");
|
|
96
|
+
if (progress.state === "cancelled") throw abortError();
|
|
97
|
+
return progress;
|
|
98
|
+
} catch (error) {
|
|
99
|
+
if (progress.state === "building") await client.cancelVectorBuild(field, progress.id);
|
|
100
|
+
throw error;
|
|
101
|
+
}
|
|
102
|
+
},
|
|
103
|
+
measureVectorRecall: async (field, queries, topK, filter, options = {}) => {
|
|
104
|
+
integer(topK, "topK", 1);
|
|
105
|
+
return send({ op: "recall", field, queries: queries.map(vectorValues), topK, filter, options: queryOptions(options) });
|
|
106
|
+
}
|
|
107
|
+
};
|
|
108
|
+
return client;
|
|
109
|
+
}
|
|
110
|
+
function vectorIndexRequest(field, options) {
|
|
111
|
+
integer(options.dimensions, "dimensions", 1);
|
|
112
|
+
if (options.metric && !["cosine", "dot", "euclidean"].includes(options.metric)) throw new Error("invalid vector metric");
|
|
113
|
+
if (options.indexType && !["flat", "hnsw"].includes(options.indexType)) throw new Error("invalid vector indexType");
|
|
114
|
+
if (options.quantization && options.quantization !== "none" && options.indexType !== "hnsw") throw new Error("quantization requires an HNSW index");
|
|
115
|
+
if (options.indexType === "hnsw") {
|
|
116
|
+
const m = options.hnswM ?? 32;
|
|
117
|
+
const efConstruction = options.hnswEfConstruction ?? 200;
|
|
118
|
+
integer(m, "hnswM", 2, 128);
|
|
119
|
+
integer(efConstruction, "hnswEfConstruction", m, 1e5);
|
|
120
|
+
if (options.metric === "dot") throw new Error("HNSW requires cosine or euclidean");
|
|
121
|
+
if (options.quantization === "binary" && options.metric && options.metric !== "cosine") throw new Error("binary quantization requires cosine");
|
|
122
|
+
}
|
|
123
|
+
return {
|
|
124
|
+
op: "create",
|
|
125
|
+
field,
|
|
126
|
+
dimensions: options.dimensions,
|
|
127
|
+
metric: options.metric,
|
|
128
|
+
options: options.indexType === "hnsw" ? { m: options.hnswM ?? 32, efConstruction: options.hnswEfConstruction ?? 200, quantization: options.quantization ?? "none" } : null
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
|
|
45
132
|
// src/webhook.ts
|
|
46
133
|
var METHOD = {
|
|
47
134
|
insert: "POST",
|
|
@@ -784,7 +871,9 @@ async function createBrowserDB(dbName, webhook, config, passphrase, migrations)
|
|
|
784
871
|
}
|
|
785
872
|
function wrapCollection(name, opts) {
|
|
786
873
|
const s = JSON.stringify;
|
|
874
|
+
const vectorCommand = async (request) => JSON.parse(await proxy.send("vectorCommand", { collection: name, requestJson: JSON.stringify(request) }));
|
|
787
875
|
const wrapped = {
|
|
876
|
+
...createVectorClient(vectorCommand),
|
|
788
877
|
insert: (doc) => proxy.send("insert", { collection: name, docJson: s(doc) }),
|
|
789
878
|
insertMany: async (docs) => {
|
|
790
879
|
const json = await proxy.send("insertMany", {
|
|
@@ -836,25 +925,21 @@ async function createBrowserDB(dbName, webhook, config, passphrase, migrations)
|
|
|
836
925
|
dropCompoundIndex: (fields) => proxy.send("dropCompoundIndex", { collection: name, fieldsJson: JSON.stringify(fields) }),
|
|
837
926
|
createFtsIndex: (field) => proxy.send("createFtsIndex", { collection: name, field }),
|
|
838
927
|
dropFtsIndex: (field) => proxy.send("dropFtsIndex", { collection: name, field }),
|
|
839
|
-
createVectorIndex: (field, options) => {
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
field,
|
|
844
|
-
dimensions: options.dimensions,
|
|
845
|
-
metric: options.metric,
|
|
846
|
-
indexType: null,
|
|
847
|
-
hnswM: null,
|
|
848
|
-
hnswEfConstruction: null
|
|
849
|
-
});
|
|
928
|
+
createVectorIndex: async (field, options) => {
|
|
929
|
+
const request = vectorIndexRequest(field, options);
|
|
930
|
+
await vectorCommand({ ...request, deferBuild: true });
|
|
931
|
+
if (request.options) await createVectorClient(vectorCommand).rebuildVectorIndex(field, request.options);
|
|
850
932
|
},
|
|
851
933
|
dropVectorIndex: (field) => proxy.send("dropVectorIndex", { collection: name, field }),
|
|
852
|
-
upgradeVectorIndex: (
|
|
934
|
+
upgradeVectorIndex: async (field) => {
|
|
935
|
+
await createVectorClient(vectorCommand).rebuildVectorIndex(field);
|
|
936
|
+
},
|
|
853
937
|
listIndexes: async () => {
|
|
854
938
|
const json = await proxy.send("listIndexes", { collection: name });
|
|
855
939
|
return JSON.parse(json);
|
|
856
940
|
},
|
|
857
|
-
findNearest: async (field, vector, topK, filter) => {
|
|
941
|
+
findNearest: async (field, vector, topK, filter, options) => {
|
|
942
|
+
if (options) return (await createVectorClient(vectorCommand).searchVectors(field, vector, topK, filter, options)).hits;
|
|
858
943
|
const json = await proxy.send("findNearest", {
|
|
859
944
|
collection: name,
|
|
860
945
|
field,
|
|
@@ -1015,7 +1100,9 @@ async function createNodeDB(dbName, webhook, config, passphrase, migrations) {
|
|
|
1015
1100
|
const db = TalaDBNode.open(dbName, configJson, passphrase ?? null);
|
|
1016
1101
|
function wrapCollection(name, opts) {
|
|
1017
1102
|
const col = db.collection(name);
|
|
1103
|
+
const vectorCommand = async (request) => col.vectorCommandAsync ? col.vectorCommandAsync(request) : col.vectorCommand(request);
|
|
1018
1104
|
const wrapped = {
|
|
1105
|
+
...createVectorClient(vectorCommand),
|
|
1019
1106
|
insert: async (doc) => col.insertAsync ? col.insertAsync(doc) : col.insert(doc),
|
|
1020
1107
|
insertMany: async (docs) => col.insertManyAsync ? col.insertManyAsync(docs) : col.insertMany(docs),
|
|
1021
1108
|
find: async (filter) => col.findAsync ? col.findAsync(filter ?? null) : col.find(filter ?? null),
|
|
@@ -1032,14 +1119,19 @@ async function createNodeDB(dbName, webhook, config, passphrase, migrations) {
|
|
|
1032
1119
|
dropCompoundIndex: async (fields) => col.dropCompoundIndex(fields),
|
|
1033
1120
|
createFtsIndex: async (field) => col.createFtsIndex(field),
|
|
1034
1121
|
dropFtsIndex: async (field) => col.dropFtsIndex(field),
|
|
1035
|
-
createVectorIndex: async (field, options) =>
|
|
1122
|
+
createVectorIndex: async (field, options) => {
|
|
1123
|
+
const request = vectorIndexRequest(field, options);
|
|
1124
|
+
await vectorCommand({ ...request, deferBuild: true });
|
|
1125
|
+
if (request.options) await createVectorClient(vectorCommand).rebuildVectorIndex(field, request.options);
|
|
1126
|
+
},
|
|
1036
1127
|
dropVectorIndex: async (field) => col.dropVectorIndex(field),
|
|
1037
1128
|
upgradeVectorIndex: async (field) => col.upgradeVectorIndex(field),
|
|
1038
1129
|
listIndexes: async () => {
|
|
1039
1130
|
const json = col.listIndexes();
|
|
1040
1131
|
return JSON.parse(json);
|
|
1041
1132
|
},
|
|
1042
|
-
findNearest: async (field, vector, topK, filter) => {
|
|
1133
|
+
findNearest: async (field, vector, topK, filter, options) => {
|
|
1134
|
+
if (options) return (await createVectorClient(vectorCommand).searchVectors(field, vector, topK, filter, options)).hits;
|
|
1043
1135
|
const raw = await col.findNearest(field, vector, topK, filter ?? null);
|
|
1044
1136
|
return raw;
|
|
1045
1137
|
},
|
|
@@ -1118,7 +1210,9 @@ async function createNativeDB(_dbName, webhook, migrations) {
|
|
|
1118
1210
|
return result;
|
|
1119
1211
|
};
|
|
1120
1212
|
function wrapCollection(name, opts) {
|
|
1213
|
+
const vectorCommand = async (request) => call("vectorCommand", name, request);
|
|
1121
1214
|
const wrapped = {
|
|
1215
|
+
...createVectorClient(vectorCommand),
|
|
1122
1216
|
insert: async (doc) => await call("insert", name, doc),
|
|
1123
1217
|
insertMany: async (docs) => await call("insertMany", name, docs),
|
|
1124
1218
|
find: async (filter) => await call("find", name, filter ?? {}),
|
|
@@ -1136,17 +1230,15 @@ async function createNativeDB(_dbName, webhook, migrations) {
|
|
|
1136
1230
|
createFtsIndex: async (field) => await call("createFtsIndex", name, field),
|
|
1137
1231
|
dropFtsIndex: async (field) => await call("dropFtsIndex", name, field),
|
|
1138
1232
|
createVectorIndex: async (field, options) => {
|
|
1139
|
-
const
|
|
1140
|
-
|
|
1141
|
-
if (options
|
|
1142
|
-
opts2.hnsw = { m: options.hnswM ?? 32, ef_construction: options.hnswEfConstruction ?? 200 };
|
|
1143
|
-
}
|
|
1144
|
-
return await call("createVectorIndex", name, field, options.dimensions, opts2);
|
|
1233
|
+
const request = vectorIndexRequest(field, options);
|
|
1234
|
+
await vectorCommand({ ...request, deferBuild: true });
|
|
1235
|
+
if (request.options) await createVectorClient(vectorCommand).rebuildVectorIndex(field, request.options);
|
|
1145
1236
|
},
|
|
1146
1237
|
dropVectorIndex: async (field) => await call("dropVectorIndex", name, field),
|
|
1147
1238
|
upgradeVectorIndex: async (field) => await call("upgradeVectorIndex", name, field),
|
|
1148
1239
|
listIndexes: async () => call("listIndexes", name),
|
|
1149
|
-
findNearest: async (field, vector, topK, filter) => {
|
|
1240
|
+
findNearest: async (field, vector, topK, filter, options) => {
|
|
1241
|
+
if (options) return (await createVectorClient(vectorCommand).searchVectors(field, vector, topK, filter, options)).hits;
|
|
1150
1242
|
const raw = await call("findNearest", name, field, vector, topK, filter ?? null);
|
|
1151
1243
|
return raw;
|
|
1152
1244
|
},
|
|
@@ -1174,6 +1266,9 @@ async function createNativeDB(_dbName, webhook, migrations) {
|
|
|
1174
1266
|
flush: native.callAsync || native.flush ? async () => {
|
|
1175
1267
|
await call("flush");
|
|
1176
1268
|
} : void 0,
|
|
1269
|
+
rebuildVectorIndexes: native.callAsync || native.rebuildVectorIndexes ? async () => {
|
|
1270
|
+
await call("rebuildVectorIndexes");
|
|
1271
|
+
} : void 0,
|
|
1177
1272
|
// One process owns the file — there is no other tab to defer to.
|
|
1178
1273
|
isPrimary: async () => true
|
|
1179
1274
|
};
|
|
@@ -1262,6 +1357,7 @@ function attachWebhook(db, webhook) {
|
|
|
1262
1357
|
0 && (module.exports = {
|
|
1263
1358
|
TalaDbValidationError,
|
|
1264
1359
|
applySchema,
|
|
1360
|
+
createVectorClient,
|
|
1265
1361
|
createWebhookDispatcher,
|
|
1266
1362
|
decorateCollection,
|
|
1267
1363
|
deriveDocId,
|
package/dist/index.mjs
CHANGED
|
@@ -1,3 +1,89 @@
|
|
|
1
|
+
// src/vector-client.ts
|
|
2
|
+
function integer(value, name, min = 0, max = 4294967295) {
|
|
3
|
+
if (!Number.isSafeInteger(value) || value < min || value > max) throw new Error(`${name} must be an integer between ${min} and ${max}`);
|
|
4
|
+
}
|
|
5
|
+
function vectorValues(vector) {
|
|
6
|
+
const values = Array.from(vector);
|
|
7
|
+
if (values.length === 0 || values.some((v) => !Number.isFinite(v))) throw new Error("vector must contain finite numbers");
|
|
8
|
+
return values;
|
|
9
|
+
}
|
|
10
|
+
function queryOptions(options) {
|
|
11
|
+
if (options.efSearch !== void 0) integer(options.efSearch, "efSearch", 1);
|
|
12
|
+
if (options.offset !== void 0) integer(options.offset, "offset");
|
|
13
|
+
if (options.groupSize !== void 0) integer(options.groupSize, "groupSize", 1);
|
|
14
|
+
if (options.oversampling !== void 0) integer(options.oversampling, "oversampling", 1, 100);
|
|
15
|
+
if (options.scoreThreshold !== void 0 && !Number.isFinite(options.scoreThreshold)) throw new Error("scoreThreshold must be finite");
|
|
16
|
+
return options;
|
|
17
|
+
}
|
|
18
|
+
function abortError() {
|
|
19
|
+
const error = new Error("Vector index rebuild cancelled");
|
|
20
|
+
error.name = "AbortError";
|
|
21
|
+
return error;
|
|
22
|
+
}
|
|
23
|
+
function createVectorClient(send) {
|
|
24
|
+
const client = {
|
|
25
|
+
searchVectors: async (field, vector, topK, filter, options = {}) => {
|
|
26
|
+
integer(topK, "topK");
|
|
27
|
+
return send({ op: "search", field, query: vectorValues(vector), topK, filter, options: queryOptions(options) });
|
|
28
|
+
},
|
|
29
|
+
findWithin: (field, vector, scoreThreshold, filter) => client.searchVectors(field, vector, 4294967295, filter, { mode: "exact", scoreThreshold }),
|
|
30
|
+
vectorIndexStatus: (field) => send({ op: "status", field }),
|
|
31
|
+
beginVectorBuild: (field, options) => send({ op: "beginBuild", field, options }),
|
|
32
|
+
stepVectorBuild: async (field, id, batchSize = 32) => {
|
|
33
|
+
integer(batchSize, "batchSize", 1, 1024);
|
|
34
|
+
return send({ op: "stepBuild", field, id, batchSize });
|
|
35
|
+
},
|
|
36
|
+
cancelVectorBuild: (field, id) => send({ op: "cancelBuild", field, id }),
|
|
37
|
+
rebuildVectorIndex: async (field, options = {}) => {
|
|
38
|
+
const { signal, onProgress, batchSize = 32, ...graphOptions } = options;
|
|
39
|
+
integer(batchSize, "batchSize", 1, 1024);
|
|
40
|
+
if (signal?.aborted) throw abortError();
|
|
41
|
+
let progress = await client.beginVectorBuild(field, Object.keys(graphOptions).length ? graphOptions : void 0);
|
|
42
|
+
try {
|
|
43
|
+
onProgress?.(progress);
|
|
44
|
+
while (progress.state === "building") {
|
|
45
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
46
|
+
if (signal?.aborted) throw abortError();
|
|
47
|
+
progress = await client.stepVectorBuild(field, progress.id, batchSize);
|
|
48
|
+
onProgress?.(progress);
|
|
49
|
+
}
|
|
50
|
+
if (progress.state === "failed") throw new Error(progress.error ?? "Vector rebuild failed");
|
|
51
|
+
if (progress.state === "cancelled") throw abortError();
|
|
52
|
+
return progress;
|
|
53
|
+
} catch (error) {
|
|
54
|
+
if (progress.state === "building") await client.cancelVectorBuild(field, progress.id);
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
measureVectorRecall: async (field, queries, topK, filter, options = {}) => {
|
|
59
|
+
integer(topK, "topK", 1);
|
|
60
|
+
return send({ op: "recall", field, queries: queries.map(vectorValues), topK, filter, options: queryOptions(options) });
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
return client;
|
|
64
|
+
}
|
|
65
|
+
function vectorIndexRequest(field, options) {
|
|
66
|
+
integer(options.dimensions, "dimensions", 1);
|
|
67
|
+
if (options.metric && !["cosine", "dot", "euclidean"].includes(options.metric)) throw new Error("invalid vector metric");
|
|
68
|
+
if (options.indexType && !["flat", "hnsw"].includes(options.indexType)) throw new Error("invalid vector indexType");
|
|
69
|
+
if (options.quantization && options.quantization !== "none" && options.indexType !== "hnsw") throw new Error("quantization requires an HNSW index");
|
|
70
|
+
if (options.indexType === "hnsw") {
|
|
71
|
+
const m = options.hnswM ?? 32;
|
|
72
|
+
const efConstruction = options.hnswEfConstruction ?? 200;
|
|
73
|
+
integer(m, "hnswM", 2, 128);
|
|
74
|
+
integer(efConstruction, "hnswEfConstruction", m, 1e5);
|
|
75
|
+
if (options.metric === "dot") throw new Error("HNSW requires cosine or euclidean");
|
|
76
|
+
if (options.quantization === "binary" && options.metric && options.metric !== "cosine") throw new Error("binary quantization requires cosine");
|
|
77
|
+
}
|
|
78
|
+
return {
|
|
79
|
+
op: "create",
|
|
80
|
+
field,
|
|
81
|
+
dimensions: options.dimensions,
|
|
82
|
+
metric: options.metric,
|
|
83
|
+
options: options.indexType === "hnsw" ? { m: options.hnswM ?? 32, efConstruction: options.hnswEfConstruction ?? 200, quantization: options.quantization ?? "none" } : null
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
1
87
|
// src/webhook.ts
|
|
2
88
|
var METHOD = {
|
|
3
89
|
insert: "POST",
|
|
@@ -739,7 +825,9 @@ async function createBrowserDB(dbName, webhook, config, passphrase, migrations)
|
|
|
739
825
|
}
|
|
740
826
|
function wrapCollection(name, opts) {
|
|
741
827
|
const s = JSON.stringify;
|
|
828
|
+
const vectorCommand = async (request) => JSON.parse(await proxy.send("vectorCommand", { collection: name, requestJson: JSON.stringify(request) }));
|
|
742
829
|
const wrapped = {
|
|
830
|
+
...createVectorClient(vectorCommand),
|
|
743
831
|
insert: (doc) => proxy.send("insert", { collection: name, docJson: s(doc) }),
|
|
744
832
|
insertMany: async (docs) => {
|
|
745
833
|
const json = await proxy.send("insertMany", {
|
|
@@ -791,25 +879,21 @@ async function createBrowserDB(dbName, webhook, config, passphrase, migrations)
|
|
|
791
879
|
dropCompoundIndex: (fields) => proxy.send("dropCompoundIndex", { collection: name, fieldsJson: JSON.stringify(fields) }),
|
|
792
880
|
createFtsIndex: (field) => proxy.send("createFtsIndex", { collection: name, field }),
|
|
793
881
|
dropFtsIndex: (field) => proxy.send("dropFtsIndex", { collection: name, field }),
|
|
794
|
-
createVectorIndex: (field, options) => {
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
field,
|
|
799
|
-
dimensions: options.dimensions,
|
|
800
|
-
metric: options.metric,
|
|
801
|
-
indexType: null,
|
|
802
|
-
hnswM: null,
|
|
803
|
-
hnswEfConstruction: null
|
|
804
|
-
});
|
|
882
|
+
createVectorIndex: async (field, options) => {
|
|
883
|
+
const request = vectorIndexRequest(field, options);
|
|
884
|
+
await vectorCommand({ ...request, deferBuild: true });
|
|
885
|
+
if (request.options) await createVectorClient(vectorCommand).rebuildVectorIndex(field, request.options);
|
|
805
886
|
},
|
|
806
887
|
dropVectorIndex: (field) => proxy.send("dropVectorIndex", { collection: name, field }),
|
|
807
|
-
upgradeVectorIndex: (
|
|
888
|
+
upgradeVectorIndex: async (field) => {
|
|
889
|
+
await createVectorClient(vectorCommand).rebuildVectorIndex(field);
|
|
890
|
+
},
|
|
808
891
|
listIndexes: async () => {
|
|
809
892
|
const json = await proxy.send("listIndexes", { collection: name });
|
|
810
893
|
return JSON.parse(json);
|
|
811
894
|
},
|
|
812
|
-
findNearest: async (field, vector, topK, filter) => {
|
|
895
|
+
findNearest: async (field, vector, topK, filter, options) => {
|
|
896
|
+
if (options) return (await createVectorClient(vectorCommand).searchVectors(field, vector, topK, filter, options)).hits;
|
|
813
897
|
const json = await proxy.send("findNearest", {
|
|
814
898
|
collection: name,
|
|
815
899
|
field,
|
|
@@ -970,7 +1054,9 @@ async function createNodeDB(dbName, webhook, config, passphrase, migrations) {
|
|
|
970
1054
|
const db = TalaDBNode.open(dbName, configJson, passphrase ?? null);
|
|
971
1055
|
function wrapCollection(name, opts) {
|
|
972
1056
|
const col = db.collection(name);
|
|
1057
|
+
const vectorCommand = async (request) => col.vectorCommandAsync ? col.vectorCommandAsync(request) : col.vectorCommand(request);
|
|
973
1058
|
const wrapped = {
|
|
1059
|
+
...createVectorClient(vectorCommand),
|
|
974
1060
|
insert: async (doc) => col.insertAsync ? col.insertAsync(doc) : col.insert(doc),
|
|
975
1061
|
insertMany: async (docs) => col.insertManyAsync ? col.insertManyAsync(docs) : col.insertMany(docs),
|
|
976
1062
|
find: async (filter) => col.findAsync ? col.findAsync(filter ?? null) : col.find(filter ?? null),
|
|
@@ -987,14 +1073,19 @@ async function createNodeDB(dbName, webhook, config, passphrase, migrations) {
|
|
|
987
1073
|
dropCompoundIndex: async (fields) => col.dropCompoundIndex(fields),
|
|
988
1074
|
createFtsIndex: async (field) => col.createFtsIndex(field),
|
|
989
1075
|
dropFtsIndex: async (field) => col.dropFtsIndex(field),
|
|
990
|
-
createVectorIndex: async (field, options) =>
|
|
1076
|
+
createVectorIndex: async (field, options) => {
|
|
1077
|
+
const request = vectorIndexRequest(field, options);
|
|
1078
|
+
await vectorCommand({ ...request, deferBuild: true });
|
|
1079
|
+
if (request.options) await createVectorClient(vectorCommand).rebuildVectorIndex(field, request.options);
|
|
1080
|
+
},
|
|
991
1081
|
dropVectorIndex: async (field) => col.dropVectorIndex(field),
|
|
992
1082
|
upgradeVectorIndex: async (field) => col.upgradeVectorIndex(field),
|
|
993
1083
|
listIndexes: async () => {
|
|
994
1084
|
const json = col.listIndexes();
|
|
995
1085
|
return JSON.parse(json);
|
|
996
1086
|
},
|
|
997
|
-
findNearest: async (field, vector, topK, filter) => {
|
|
1087
|
+
findNearest: async (field, vector, topK, filter, options) => {
|
|
1088
|
+
if (options) return (await createVectorClient(vectorCommand).searchVectors(field, vector, topK, filter, options)).hits;
|
|
998
1089
|
const raw = await col.findNearest(field, vector, topK, filter ?? null);
|
|
999
1090
|
return raw;
|
|
1000
1091
|
},
|
|
@@ -1073,7 +1164,9 @@ async function createNativeDB(_dbName, webhook, migrations) {
|
|
|
1073
1164
|
return result;
|
|
1074
1165
|
};
|
|
1075
1166
|
function wrapCollection(name, opts) {
|
|
1167
|
+
const vectorCommand = async (request) => call("vectorCommand", name, request);
|
|
1076
1168
|
const wrapped = {
|
|
1169
|
+
...createVectorClient(vectorCommand),
|
|
1077
1170
|
insert: async (doc) => await call("insert", name, doc),
|
|
1078
1171
|
insertMany: async (docs) => await call("insertMany", name, docs),
|
|
1079
1172
|
find: async (filter) => await call("find", name, filter ?? {}),
|
|
@@ -1091,17 +1184,15 @@ async function createNativeDB(_dbName, webhook, migrations) {
|
|
|
1091
1184
|
createFtsIndex: async (field) => await call("createFtsIndex", name, field),
|
|
1092
1185
|
dropFtsIndex: async (field) => await call("dropFtsIndex", name, field),
|
|
1093
1186
|
createVectorIndex: async (field, options) => {
|
|
1094
|
-
const
|
|
1095
|
-
|
|
1096
|
-
if (options
|
|
1097
|
-
opts2.hnsw = { m: options.hnswM ?? 32, ef_construction: options.hnswEfConstruction ?? 200 };
|
|
1098
|
-
}
|
|
1099
|
-
return await call("createVectorIndex", name, field, options.dimensions, opts2);
|
|
1187
|
+
const request = vectorIndexRequest(field, options);
|
|
1188
|
+
await vectorCommand({ ...request, deferBuild: true });
|
|
1189
|
+
if (request.options) await createVectorClient(vectorCommand).rebuildVectorIndex(field, request.options);
|
|
1100
1190
|
},
|
|
1101
1191
|
dropVectorIndex: async (field) => await call("dropVectorIndex", name, field),
|
|
1102
1192
|
upgradeVectorIndex: async (field) => await call("upgradeVectorIndex", name, field),
|
|
1103
1193
|
listIndexes: async () => call("listIndexes", name),
|
|
1104
|
-
findNearest: async (field, vector, topK, filter) => {
|
|
1194
|
+
findNearest: async (field, vector, topK, filter, options) => {
|
|
1195
|
+
if (options) return (await createVectorClient(vectorCommand).searchVectors(field, vector, topK, filter, options)).hits;
|
|
1105
1196
|
const raw = await call("findNearest", name, field, vector, topK, filter ?? null);
|
|
1106
1197
|
return raw;
|
|
1107
1198
|
},
|
|
@@ -1129,6 +1220,9 @@ async function createNativeDB(_dbName, webhook, migrations) {
|
|
|
1129
1220
|
flush: native.callAsync || native.flush ? async () => {
|
|
1130
1221
|
await call("flush");
|
|
1131
1222
|
} : void 0,
|
|
1223
|
+
rebuildVectorIndexes: native.callAsync || native.rebuildVectorIndexes ? async () => {
|
|
1224
|
+
await call("rebuildVectorIndexes");
|
|
1225
|
+
} : void 0,
|
|
1132
1226
|
// One process owns the file — there is no other tab to defer to.
|
|
1133
1227
|
isPrimary: async () => true
|
|
1134
1228
|
};
|
|
@@ -1216,6 +1310,7 @@ function attachWebhook(db, webhook) {
|
|
|
1216
1310
|
export {
|
|
1217
1311
|
TalaDbValidationError,
|
|
1218
1312
|
applySchema,
|
|
1313
|
+
createVectorClient,
|
|
1219
1314
|
createWebhookDispatcher,
|
|
1220
1315
|
decorateCollection,
|
|
1221
1316
|
deriveDocId,
|
|
@@ -1,3 +1,89 @@
|
|
|
1
|
+
// src/vector-client.ts
|
|
2
|
+
function integer(value, name, min = 0, max = 4294967295) {
|
|
3
|
+
if (!Number.isSafeInteger(value) || value < min || value > max) throw new Error(`${name} must be an integer between ${min} and ${max}`);
|
|
4
|
+
}
|
|
5
|
+
function vectorValues(vector) {
|
|
6
|
+
const values = Array.from(vector);
|
|
7
|
+
if (values.length === 0 || values.some((v) => !Number.isFinite(v))) throw new Error("vector must contain finite numbers");
|
|
8
|
+
return values;
|
|
9
|
+
}
|
|
10
|
+
function queryOptions(options) {
|
|
11
|
+
if (options.efSearch !== void 0) integer(options.efSearch, "efSearch", 1);
|
|
12
|
+
if (options.offset !== void 0) integer(options.offset, "offset");
|
|
13
|
+
if (options.groupSize !== void 0) integer(options.groupSize, "groupSize", 1);
|
|
14
|
+
if (options.oversampling !== void 0) integer(options.oversampling, "oversampling", 1, 100);
|
|
15
|
+
if (options.scoreThreshold !== void 0 && !Number.isFinite(options.scoreThreshold)) throw new Error("scoreThreshold must be finite");
|
|
16
|
+
return options;
|
|
17
|
+
}
|
|
18
|
+
function abortError() {
|
|
19
|
+
const error = new Error("Vector index rebuild cancelled");
|
|
20
|
+
error.name = "AbortError";
|
|
21
|
+
return error;
|
|
22
|
+
}
|
|
23
|
+
function createVectorClient(send) {
|
|
24
|
+
const client = {
|
|
25
|
+
searchVectors: async (field, vector, topK, filter, options = {}) => {
|
|
26
|
+
integer(topK, "topK");
|
|
27
|
+
return send({ op: "search", field, query: vectorValues(vector), topK, filter, options: queryOptions(options) });
|
|
28
|
+
},
|
|
29
|
+
findWithin: (field, vector, scoreThreshold, filter) => client.searchVectors(field, vector, 4294967295, filter, { mode: "exact", scoreThreshold }),
|
|
30
|
+
vectorIndexStatus: (field) => send({ op: "status", field }),
|
|
31
|
+
beginVectorBuild: (field, options) => send({ op: "beginBuild", field, options }),
|
|
32
|
+
stepVectorBuild: async (field, id, batchSize = 32) => {
|
|
33
|
+
integer(batchSize, "batchSize", 1, 1024);
|
|
34
|
+
return send({ op: "stepBuild", field, id, batchSize });
|
|
35
|
+
},
|
|
36
|
+
cancelVectorBuild: (field, id) => send({ op: "cancelBuild", field, id }),
|
|
37
|
+
rebuildVectorIndex: async (field, options = {}) => {
|
|
38
|
+
const { signal, onProgress, batchSize = 32, ...graphOptions } = options;
|
|
39
|
+
integer(batchSize, "batchSize", 1, 1024);
|
|
40
|
+
if (signal?.aborted) throw abortError();
|
|
41
|
+
let progress = await client.beginVectorBuild(field, Object.keys(graphOptions).length ? graphOptions : void 0);
|
|
42
|
+
try {
|
|
43
|
+
onProgress?.(progress);
|
|
44
|
+
while (progress.state === "building") {
|
|
45
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
46
|
+
if (signal?.aborted) throw abortError();
|
|
47
|
+
progress = await client.stepVectorBuild(field, progress.id, batchSize);
|
|
48
|
+
onProgress?.(progress);
|
|
49
|
+
}
|
|
50
|
+
if (progress.state === "failed") throw new Error(progress.error ?? "Vector rebuild failed");
|
|
51
|
+
if (progress.state === "cancelled") throw abortError();
|
|
52
|
+
return progress;
|
|
53
|
+
} catch (error) {
|
|
54
|
+
if (progress.state === "building") await client.cancelVectorBuild(field, progress.id);
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
measureVectorRecall: async (field, queries, topK, filter, options = {}) => {
|
|
59
|
+
integer(topK, "topK", 1);
|
|
60
|
+
return send({ op: "recall", field, queries: queries.map(vectorValues), topK, filter, options: queryOptions(options) });
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
return client;
|
|
64
|
+
}
|
|
65
|
+
function vectorIndexRequest(field, options) {
|
|
66
|
+
integer(options.dimensions, "dimensions", 1);
|
|
67
|
+
if (options.metric && !["cosine", "dot", "euclidean"].includes(options.metric)) throw new Error("invalid vector metric");
|
|
68
|
+
if (options.indexType && !["flat", "hnsw"].includes(options.indexType)) throw new Error("invalid vector indexType");
|
|
69
|
+
if (options.quantization && options.quantization !== "none" && options.indexType !== "hnsw") throw new Error("quantization requires an HNSW index");
|
|
70
|
+
if (options.indexType === "hnsw") {
|
|
71
|
+
const m = options.hnswM ?? 32;
|
|
72
|
+
const efConstruction = options.hnswEfConstruction ?? 200;
|
|
73
|
+
integer(m, "hnswM", 2, 128);
|
|
74
|
+
integer(efConstruction, "hnswEfConstruction", m, 1e5);
|
|
75
|
+
if (options.metric === "dot") throw new Error("HNSW requires cosine or euclidean");
|
|
76
|
+
if (options.quantization === "binary" && options.metric && options.metric !== "cosine") throw new Error("binary quantization requires cosine");
|
|
77
|
+
}
|
|
78
|
+
return {
|
|
79
|
+
op: "create",
|
|
80
|
+
field,
|
|
81
|
+
dimensions: options.dimensions,
|
|
82
|
+
metric: options.metric,
|
|
83
|
+
options: options.indexType === "hnsw" ? { m: options.hnswM ?? 32, efConstruction: options.hnswEfConstruction ?? 200, quantization: options.quantization ?? "none" } : null
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
1
87
|
// src/config.browser.ts
|
|
2
88
|
var ENDPOINT_FIELDS = [
|
|
3
89
|
"endpoint",
|
|
@@ -710,7 +796,9 @@ async function createBrowserDB(dbName, webhook, config, passphrase, migrations)
|
|
|
710
796
|
}
|
|
711
797
|
function wrapCollection(name, opts) {
|
|
712
798
|
const s = JSON.stringify;
|
|
799
|
+
const vectorCommand = async (request) => JSON.parse(await proxy.send("vectorCommand", { collection: name, requestJson: JSON.stringify(request) }));
|
|
713
800
|
const wrapped = {
|
|
801
|
+
...createVectorClient(vectorCommand),
|
|
714
802
|
insert: (doc) => proxy.send("insert", { collection: name, docJson: s(doc) }),
|
|
715
803
|
insertMany: async (docs) => {
|
|
716
804
|
const json = await proxy.send("insertMany", {
|
|
@@ -762,25 +850,21 @@ async function createBrowserDB(dbName, webhook, config, passphrase, migrations)
|
|
|
762
850
|
dropCompoundIndex: (fields) => proxy.send("dropCompoundIndex", { collection: name, fieldsJson: JSON.stringify(fields) }),
|
|
763
851
|
createFtsIndex: (field) => proxy.send("createFtsIndex", { collection: name, field }),
|
|
764
852
|
dropFtsIndex: (field) => proxy.send("dropFtsIndex", { collection: name, field }),
|
|
765
|
-
createVectorIndex: (field, options) => {
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
field,
|
|
770
|
-
dimensions: options.dimensions,
|
|
771
|
-
metric: options.metric,
|
|
772
|
-
indexType: null,
|
|
773
|
-
hnswM: null,
|
|
774
|
-
hnswEfConstruction: null
|
|
775
|
-
});
|
|
853
|
+
createVectorIndex: async (field, options) => {
|
|
854
|
+
const request = vectorIndexRequest(field, options);
|
|
855
|
+
await vectorCommand({ ...request, deferBuild: true });
|
|
856
|
+
if (request.options) await createVectorClient(vectorCommand).rebuildVectorIndex(field, request.options);
|
|
776
857
|
},
|
|
777
858
|
dropVectorIndex: (field) => proxy.send("dropVectorIndex", { collection: name, field }),
|
|
778
|
-
upgradeVectorIndex: (
|
|
859
|
+
upgradeVectorIndex: async (field) => {
|
|
860
|
+
await createVectorClient(vectorCommand).rebuildVectorIndex(field);
|
|
861
|
+
},
|
|
779
862
|
listIndexes: async () => {
|
|
780
863
|
const json = await proxy.send("listIndexes", { collection: name });
|
|
781
864
|
return JSON.parse(json);
|
|
782
865
|
},
|
|
783
|
-
findNearest: async (field, vector, topK, filter) => {
|
|
866
|
+
findNearest: async (field, vector, topK, filter, options) => {
|
|
867
|
+
if (options) return (await createVectorClient(vectorCommand).searchVectors(field, vector, topK, filter, options)).hits;
|
|
784
868
|
const json = await proxy.send("findNearest", {
|
|
785
869
|
collection: name,
|
|
786
870
|
field,
|
|
@@ -941,7 +1025,9 @@ async function createNodeDB(dbName, webhook, config, passphrase, migrations) {
|
|
|
941
1025
|
const db = TalaDBNode.open(dbName, configJson, passphrase ?? null);
|
|
942
1026
|
function wrapCollection(name, opts) {
|
|
943
1027
|
const col = db.collection(name);
|
|
1028
|
+
const vectorCommand = async (request) => col.vectorCommandAsync ? col.vectorCommandAsync(request) : col.vectorCommand(request);
|
|
944
1029
|
const wrapped = {
|
|
1030
|
+
...createVectorClient(vectorCommand),
|
|
945
1031
|
insert: async (doc) => col.insertAsync ? col.insertAsync(doc) : col.insert(doc),
|
|
946
1032
|
insertMany: async (docs) => col.insertManyAsync ? col.insertManyAsync(docs) : col.insertMany(docs),
|
|
947
1033
|
find: async (filter) => col.findAsync ? col.findAsync(filter ?? null) : col.find(filter ?? null),
|
|
@@ -958,14 +1044,19 @@ async function createNodeDB(dbName, webhook, config, passphrase, migrations) {
|
|
|
958
1044
|
dropCompoundIndex: async (fields) => col.dropCompoundIndex(fields),
|
|
959
1045
|
createFtsIndex: async (field) => col.createFtsIndex(field),
|
|
960
1046
|
dropFtsIndex: async (field) => col.dropFtsIndex(field),
|
|
961
|
-
createVectorIndex: async (field, options) =>
|
|
1047
|
+
createVectorIndex: async (field, options) => {
|
|
1048
|
+
const request = vectorIndexRequest(field, options);
|
|
1049
|
+
await vectorCommand({ ...request, deferBuild: true });
|
|
1050
|
+
if (request.options) await createVectorClient(vectorCommand).rebuildVectorIndex(field, request.options);
|
|
1051
|
+
},
|
|
962
1052
|
dropVectorIndex: async (field) => col.dropVectorIndex(field),
|
|
963
1053
|
upgradeVectorIndex: async (field) => col.upgradeVectorIndex(field),
|
|
964
1054
|
listIndexes: async () => {
|
|
965
1055
|
const json = col.listIndexes();
|
|
966
1056
|
return JSON.parse(json);
|
|
967
1057
|
},
|
|
968
|
-
findNearest: async (field, vector, topK, filter) => {
|
|
1058
|
+
findNearest: async (field, vector, topK, filter, options) => {
|
|
1059
|
+
if (options) return (await createVectorClient(vectorCommand).searchVectors(field, vector, topK, filter, options)).hits;
|
|
969
1060
|
const raw = await col.findNearest(field, vector, topK, filter ?? null);
|
|
970
1061
|
return raw;
|
|
971
1062
|
},
|
|
@@ -1044,7 +1135,9 @@ async function createNativeDB(_dbName, webhook, migrations) {
|
|
|
1044
1135
|
return result;
|
|
1045
1136
|
};
|
|
1046
1137
|
function wrapCollection(name, opts) {
|
|
1138
|
+
const vectorCommand = async (request) => call("vectorCommand", name, request);
|
|
1047
1139
|
const wrapped = {
|
|
1140
|
+
...createVectorClient(vectorCommand),
|
|
1048
1141
|
insert: async (doc) => await call("insert", name, doc),
|
|
1049
1142
|
insertMany: async (docs) => await call("insertMany", name, docs),
|
|
1050
1143
|
find: async (filter) => await call("find", name, filter ?? {}),
|
|
@@ -1062,17 +1155,15 @@ async function createNativeDB(_dbName, webhook, migrations) {
|
|
|
1062
1155
|
createFtsIndex: async (field) => await call("createFtsIndex", name, field),
|
|
1063
1156
|
dropFtsIndex: async (field) => await call("dropFtsIndex", name, field),
|
|
1064
1157
|
createVectorIndex: async (field, options) => {
|
|
1065
|
-
const
|
|
1066
|
-
|
|
1067
|
-
if (options
|
|
1068
|
-
opts2.hnsw = { m: options.hnswM ?? 32, ef_construction: options.hnswEfConstruction ?? 200 };
|
|
1069
|
-
}
|
|
1070
|
-
return await call("createVectorIndex", name, field, options.dimensions, opts2);
|
|
1158
|
+
const request = vectorIndexRequest(field, options);
|
|
1159
|
+
await vectorCommand({ ...request, deferBuild: true });
|
|
1160
|
+
if (request.options) await createVectorClient(vectorCommand).rebuildVectorIndex(field, request.options);
|
|
1071
1161
|
},
|
|
1072
1162
|
dropVectorIndex: async (field) => await call("dropVectorIndex", name, field),
|
|
1073
1163
|
upgradeVectorIndex: async (field) => await call("upgradeVectorIndex", name, field),
|
|
1074
1164
|
listIndexes: async () => call("listIndexes", name),
|
|
1075
|
-
findNearest: async (field, vector, topK, filter) => {
|
|
1165
|
+
findNearest: async (field, vector, topK, filter, options) => {
|
|
1166
|
+
if (options) return (await createVectorClient(vectorCommand).searchVectors(field, vector, topK, filter, options)).hits;
|
|
1076
1167
|
const raw = await call("findNearest", name, field, vector, topK, filter ?? null);
|
|
1077
1168
|
return raw;
|
|
1078
1169
|
},
|
|
@@ -1100,6 +1191,9 @@ async function createNativeDB(_dbName, webhook, migrations) {
|
|
|
1100
1191
|
flush: native.callAsync || native.flush ? async () => {
|
|
1101
1192
|
await call("flush");
|
|
1102
1193
|
} : void 0,
|
|
1194
|
+
rebuildVectorIndexes: native.callAsync || native.rebuildVectorIndexes ? async () => {
|
|
1195
|
+
await call("rebuildVectorIndexes");
|
|
1196
|
+
} : void 0,
|
|
1103
1197
|
// One process owns the file — there is no other tab to defer to.
|
|
1104
1198
|
isPrimary: async () => true
|
|
1105
1199
|
};
|
|
@@ -1187,6 +1281,7 @@ function attachWebhook(db, webhook) {
|
|
|
1187
1281
|
export {
|
|
1188
1282
|
TalaDbValidationError,
|
|
1189
1283
|
applySchema,
|
|
1284
|
+
createVectorClient,
|
|
1190
1285
|
createWebhookDispatcher,
|
|
1191
1286
|
decorateCollection,
|
|
1192
1287
|
deriveDocId,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "taladb",
|
|
3
|
-
"version": "0.11.
|
|
3
|
+
"version": "0.11.4",
|
|
4
4
|
"description": "The embedded vector database for on-device AI — documents, similarity search, and offline sync for browser, React Native, and Node.js",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.mjs",
|
|
@@ -62,8 +62,8 @@
|
|
|
62
62
|
"vitest": "^3.2.0"
|
|
63
63
|
},
|
|
64
64
|
"peerDependencies": {
|
|
65
|
-
"@taladb/web": "0.11.
|
|
66
|
-
"@taladb/node": "0.11.
|
|
65
|
+
"@taladb/web": "0.11.4",
|
|
66
|
+
"@taladb/node": "0.11.4"
|
|
67
67
|
},
|
|
68
68
|
"peerDependenciesMeta": {
|
|
69
69
|
"@taladb/web": {
|