ydb-qdrant 5.2.1 → 7.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/config/env.d.ts +9 -3
- package/dist/config/env.js +16 -5
- package/dist/package/api.d.ts +2 -2
- package/dist/package/api.js +2 -2
- package/dist/qdrant/QdrantTypes.d.ts +19 -0
- package/dist/qdrant/QdrantTypes.js +1 -0
- package/dist/repositories/collectionsRepo.d.ts +12 -7
- package/dist/repositories/collectionsRepo.js +157 -39
- package/dist/repositories/collectionsRepo.one-table.js +47 -129
- package/dist/repositories/pointsRepo.d.ts +5 -7
- package/dist/repositories/pointsRepo.js +6 -3
- package/dist/repositories/pointsRepo.one-table/Delete.d.ts +2 -0
- package/dist/repositories/pointsRepo.one-table/Delete.js +111 -0
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.d.ts +11 -0
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.js +32 -0
- package/dist/repositories/pointsRepo.one-table/Search/Approximate.d.ts +18 -0
- package/dist/repositories/pointsRepo.one-table/Search/Approximate.js +119 -0
- package/dist/repositories/pointsRepo.one-table/Search/Exact.d.ts +17 -0
- package/dist/repositories/pointsRepo.one-table/Search/Exact.js +101 -0
- package/dist/repositories/pointsRepo.one-table/Search/index.d.ts +8 -0
- package/dist/repositories/pointsRepo.one-table/Search/index.js +30 -0
- package/dist/repositories/pointsRepo.one-table/Upsert.d.ts +2 -0
- package/dist/repositories/pointsRepo.one-table/Upsert.js +100 -0
- package/dist/repositories/pointsRepo.one-table.d.ts +3 -13
- package/dist/repositories/pointsRepo.one-table.js +3 -403
- package/dist/routes/collections.js +61 -7
- package/dist/routes/points.js +71 -3
- package/dist/server.d.ts +1 -0
- package/dist/server.js +70 -2
- package/dist/services/CollectionService.d.ts +9 -0
- package/dist/services/CollectionService.js +13 -1
- package/dist/services/CollectionService.shared.d.ts +1 -0
- package/dist/services/CollectionService.shared.js +3 -3
- package/dist/services/PointsService.d.ts +8 -10
- package/dist/services/PointsService.js +82 -5
- package/dist/types.d.ts +85 -8
- package/dist/types.js +43 -17
- package/dist/utils/normalization.d.ts +1 -0
- package/dist/utils/normalization.js +15 -13
- package/dist/utils/retry.js +29 -19
- package/dist/utils/tenant.d.ts +2 -2
- package/dist/utils/tenant.js +21 -6
- package/dist/utils/typeGuards.d.ts +1 -0
- package/dist/utils/typeGuards.js +3 -0
- package/dist/utils/vectorBinary.js +88 -9
- package/dist/ydb/QueryDiagnostics.d.ts +6 -0
- package/dist/ydb/QueryDiagnostics.js +52 -0
- package/dist/ydb/SessionPool.d.ts +36 -0
- package/dist/ydb/SessionPool.js +248 -0
- package/dist/ydb/bulkUpsert.d.ts +6 -0
- package/dist/ydb/bulkUpsert.js +52 -0
- package/dist/ydb/client.d.ts +17 -16
- package/dist/ydb/client.js +427 -62
- package/dist/ydb/helpers.d.ts +0 -2
- package/dist/ydb/helpers.js +0 -7
- package/dist/ydb/schema.js +172 -54
- package/package.json +12 -7
- package/dist/repositories/collectionsRepo.shared.d.ts +0 -2
- package/dist/repositories/collectionsRepo.shared.js +0 -23
|
@@ -1,403 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
import { withRetry, isTransientYdbError } from "../utils/retry.js";
|
|
5
|
-
import { UPSERT_BATCH_SIZE } from "../ydb/schema.js";
|
|
6
|
-
import { CLIENT_SIDE_SERIALIZATION_ENABLED, SearchMode, UPSERT_OPERATION_TIMEOUT_MS, SEARCH_OPERATION_TIMEOUT_MS, } from "../config/env.js";
|
|
7
|
-
import { logger } from "../logging/logger.js";
|
|
8
|
-
export async function upsertPointsOneTable(tableName, points, dimension, uid) {
|
|
9
|
-
for (const p of points) {
|
|
10
|
-
const id = String(p.id);
|
|
11
|
-
if (p.vector.length !== dimension) {
|
|
12
|
-
const previewLength = Math.min(16, p.vector.length);
|
|
13
|
-
const vectorPreview = previewLength > 0 ? p.vector.slice(0, previewLength) : [];
|
|
14
|
-
logger.warn({
|
|
15
|
-
tableName,
|
|
16
|
-
uid,
|
|
17
|
-
pointId: id,
|
|
18
|
-
vectorLen: p.vector.length,
|
|
19
|
-
expectedDimension: dimension,
|
|
20
|
-
vectorPreview,
|
|
21
|
-
}, "upsertPointsOneTable: vector dimension mismatch");
|
|
22
|
-
throw new Error(`Vector dimension mismatch for id=${id}: got ${p.vector.length}, expected ${dimension}`);
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
let upserted = 0;
|
|
26
|
-
await withSession(async (s) => {
|
|
27
|
-
const settings = createExecuteQuerySettingsWithTimeout({
|
|
28
|
-
keepInCache: true,
|
|
29
|
-
idempotent: true,
|
|
30
|
-
timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
|
|
31
|
-
});
|
|
32
|
-
for (let i = 0; i < points.length; i += UPSERT_BATCH_SIZE) {
|
|
33
|
-
const batch = points.slice(i, i + UPSERT_BATCH_SIZE);
|
|
34
|
-
let ddl;
|
|
35
|
-
let params;
|
|
36
|
-
if (CLIENT_SIDE_SERIALIZATION_ENABLED) {
|
|
37
|
-
ddl = `
|
|
38
|
-
DECLARE $rows AS List<Struct<
|
|
39
|
-
uid: Utf8,
|
|
40
|
-
point_id: Utf8,
|
|
41
|
-
embedding: String,
|
|
42
|
-
embedding_quantized: String,
|
|
43
|
-
payload: JsonDocument
|
|
44
|
-
>>;
|
|
45
|
-
|
|
46
|
-
UPSERT INTO ${tableName} (uid, point_id, embedding, embedding_quantized, payload)
|
|
47
|
-
SELECT
|
|
48
|
-
uid,
|
|
49
|
-
point_id,
|
|
50
|
-
embedding,
|
|
51
|
-
embedding_quantized,
|
|
52
|
-
payload
|
|
53
|
-
FROM AS_TABLE($rows);
|
|
54
|
-
`;
|
|
55
|
-
const rowType = Types.struct({
|
|
56
|
-
uid: Types.UTF8,
|
|
57
|
-
point_id: Types.UTF8,
|
|
58
|
-
embedding: Types.BYTES,
|
|
59
|
-
embedding_quantized: Types.BYTES,
|
|
60
|
-
payload: Types.JSON_DOCUMENT,
|
|
61
|
-
});
|
|
62
|
-
const rowsValue = TypedValues.list(rowType, batch.map((p) => {
|
|
63
|
-
const binaries = buildVectorBinaryParams(p.vector);
|
|
64
|
-
return {
|
|
65
|
-
uid,
|
|
66
|
-
point_id: String(p.id),
|
|
67
|
-
embedding: binaries.float,
|
|
68
|
-
embedding_quantized: binaries.bit,
|
|
69
|
-
payload: JSON.stringify(p.payload ?? {}),
|
|
70
|
-
};
|
|
71
|
-
}));
|
|
72
|
-
params = {
|
|
73
|
-
$rows: rowsValue,
|
|
74
|
-
};
|
|
75
|
-
}
|
|
76
|
-
else {
|
|
77
|
-
ddl = `
|
|
78
|
-
DECLARE $rows AS List<Struct<
|
|
79
|
-
uid: Utf8,
|
|
80
|
-
point_id: Utf8,
|
|
81
|
-
vec: List<Float>,
|
|
82
|
-
payload: JsonDocument
|
|
83
|
-
>>;
|
|
84
|
-
|
|
85
|
-
UPSERT INTO ${tableName} (uid, point_id, embedding, embedding_quantized, payload)
|
|
86
|
-
SELECT
|
|
87
|
-
uid,
|
|
88
|
-
point_id,
|
|
89
|
-
Untag(Knn::ToBinaryStringFloat(vec), "FloatVector") AS embedding,
|
|
90
|
-
Untag(Knn::ToBinaryStringBit(vec), "BitVector") AS embedding_quantized,
|
|
91
|
-
payload
|
|
92
|
-
FROM AS_TABLE($rows);
|
|
93
|
-
`;
|
|
94
|
-
const rowType = Types.struct({
|
|
95
|
-
uid: Types.UTF8,
|
|
96
|
-
point_id: Types.UTF8,
|
|
97
|
-
vec: Types.list(Types.FLOAT),
|
|
98
|
-
payload: Types.JSON_DOCUMENT,
|
|
99
|
-
});
|
|
100
|
-
const rowsValue = TypedValues.list(rowType, batch.map((p) => ({
|
|
101
|
-
uid,
|
|
102
|
-
point_id: String(p.id),
|
|
103
|
-
vec: p.vector,
|
|
104
|
-
payload: JSON.stringify(p.payload ?? {}),
|
|
105
|
-
})));
|
|
106
|
-
params = {
|
|
107
|
-
$rows: rowsValue,
|
|
108
|
-
};
|
|
109
|
-
}
|
|
110
|
-
logger.debug({
|
|
111
|
-
tableName,
|
|
112
|
-
mode: CLIENT_SIDE_SERIALIZATION_ENABLED
|
|
113
|
-
? "one_table_upsert_client_side_serialization"
|
|
114
|
-
: "one_table_upsert_server_side_knn",
|
|
115
|
-
batchSize: batch.length,
|
|
116
|
-
yql: ddl,
|
|
117
|
-
params: {
|
|
118
|
-
rows: batch.map((p) => ({
|
|
119
|
-
uid,
|
|
120
|
-
point_id: String(p.id),
|
|
121
|
-
vectorLength: p.vector.length,
|
|
122
|
-
vectorPreview: p.vector.slice(0, 3),
|
|
123
|
-
payload: p.payload ?? {},
|
|
124
|
-
})),
|
|
125
|
-
},
|
|
126
|
-
}, "one_table upsert: executing YQL");
|
|
127
|
-
await withRetry(() => s.executeQuery(ddl, params, undefined, settings), {
|
|
128
|
-
isTransient: isTransientYdbError,
|
|
129
|
-
context: { tableName, batchSize: batch.length },
|
|
130
|
-
});
|
|
131
|
-
upserted += batch.length;
|
|
132
|
-
}
|
|
133
|
-
});
|
|
134
|
-
return upserted;
|
|
135
|
-
}
|
|
136
|
-
async function searchPointsOneTableExact(tableName, queryVector, top, withPayload, distance, dimension, uid) {
|
|
137
|
-
if (queryVector.length !== dimension) {
|
|
138
|
-
throw new Error(`Vector dimension mismatch: got ${queryVector.length}, expected ${dimension}`);
|
|
139
|
-
}
|
|
140
|
-
const { fn, order } = mapDistanceToKnnFn(distance);
|
|
141
|
-
const results = await withSession(async (s) => {
|
|
142
|
-
let yql;
|
|
143
|
-
let params;
|
|
144
|
-
if (CLIENT_SIDE_SERIALIZATION_ENABLED) {
|
|
145
|
-
const binaries = buildVectorBinaryParams(queryVector);
|
|
146
|
-
yql = `
|
|
147
|
-
DECLARE $qbinf AS String;
|
|
148
|
-
DECLARE $k AS Uint32;
|
|
149
|
-
DECLARE $uid AS Utf8;
|
|
150
|
-
SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
|
|
151
|
-
FROM ${tableName}
|
|
152
|
-
WHERE uid = $uid
|
|
153
|
-
ORDER BY score ${order}
|
|
154
|
-
LIMIT $k;
|
|
155
|
-
`;
|
|
156
|
-
params = {
|
|
157
|
-
$qbinf: typeof TypedValues.bytes === "function"
|
|
158
|
-
? TypedValues.bytes(binaries.float)
|
|
159
|
-
: binaries.float,
|
|
160
|
-
$k: TypedValues.uint32(top),
|
|
161
|
-
$uid: TypedValues.utf8(uid),
|
|
162
|
-
};
|
|
163
|
-
}
|
|
164
|
-
else {
|
|
165
|
-
const qf = buildVectorParam(queryVector);
|
|
166
|
-
yql = `
|
|
167
|
-
DECLARE $qf AS List<Float>;
|
|
168
|
-
DECLARE $k AS Uint32;
|
|
169
|
-
DECLARE $uid AS Utf8;
|
|
170
|
-
$qbinf = Knn::ToBinaryStringFloat($qf);
|
|
171
|
-
SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
|
|
172
|
-
FROM ${tableName}
|
|
173
|
-
WHERE uid = $uid
|
|
174
|
-
ORDER BY score ${order}
|
|
175
|
-
LIMIT $k;
|
|
176
|
-
`;
|
|
177
|
-
params = {
|
|
178
|
-
$qf: qf,
|
|
179
|
-
$k: TypedValues.uint32(top),
|
|
180
|
-
$uid: TypedValues.utf8(uid),
|
|
181
|
-
};
|
|
182
|
-
}
|
|
183
|
-
logger.debug({
|
|
184
|
-
tableName,
|
|
185
|
-
distance,
|
|
186
|
-
top,
|
|
187
|
-
withPayload,
|
|
188
|
-
mode: CLIENT_SIDE_SERIALIZATION_ENABLED
|
|
189
|
-
? "one_table_exact_client_side_serialization"
|
|
190
|
-
: "one_table_exact",
|
|
191
|
-
yql,
|
|
192
|
-
params: {
|
|
193
|
-
uid,
|
|
194
|
-
top,
|
|
195
|
-
vectorLength: queryVector.length,
|
|
196
|
-
vectorPreview: queryVector.slice(0, 3),
|
|
197
|
-
},
|
|
198
|
-
}, "one_table search (exact): executing YQL");
|
|
199
|
-
const settings = createExecuteQuerySettingsWithTimeout({
|
|
200
|
-
keepInCache: true,
|
|
201
|
-
idempotent: true,
|
|
202
|
-
timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
|
|
203
|
-
});
|
|
204
|
-
const rs = await s.executeQuery(yql, params, undefined, settings);
|
|
205
|
-
const rowset = rs.resultSets?.[0];
|
|
206
|
-
const rows = (rowset?.rows ?? []);
|
|
207
|
-
return rows.map((row) => {
|
|
208
|
-
const id = row.items?.[0]?.textValue;
|
|
209
|
-
if (typeof id !== "string") {
|
|
210
|
-
throw new Error("point_id is missing in YDB search result");
|
|
211
|
-
}
|
|
212
|
-
let payload;
|
|
213
|
-
let scoreIdx = 1;
|
|
214
|
-
if (withPayload) {
|
|
215
|
-
const payloadText = row.items?.[1]?.textValue;
|
|
216
|
-
if (payloadText) {
|
|
217
|
-
try {
|
|
218
|
-
payload = JSON.parse(payloadText);
|
|
219
|
-
}
|
|
220
|
-
catch {
|
|
221
|
-
payload = undefined;
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
scoreIdx = 2;
|
|
225
|
-
}
|
|
226
|
-
const score = Number(row.items?.[scoreIdx]?.floatValue ?? row.items?.[scoreIdx]?.textValue);
|
|
227
|
-
return { id, score, ...(payload ? { payload } : {}) };
|
|
228
|
-
});
|
|
229
|
-
});
|
|
230
|
-
return results;
|
|
231
|
-
}
|
|
232
|
-
async function searchPointsOneTableApproximate(tableName, queryVector, top, withPayload, distance, dimension, uid, overfetchMultiplier) {
|
|
233
|
-
if (queryVector.length !== dimension) {
|
|
234
|
-
throw new Error(`Vector dimension mismatch: got ${queryVector.length}, expected ${dimension}`);
|
|
235
|
-
}
|
|
236
|
-
const { fn, order } = mapDistanceToKnnFn(distance);
|
|
237
|
-
const { fn: bitFn, order: bitOrder } = mapDistanceToBitKnnFn(distance);
|
|
238
|
-
const safeTop = top > 0 ? top : 1;
|
|
239
|
-
const rawCandidateLimit = safeTop * overfetchMultiplier;
|
|
240
|
-
const candidateLimit = Math.max(safeTop, rawCandidateLimit);
|
|
241
|
-
const results = await withSession(async (s) => {
|
|
242
|
-
let yql;
|
|
243
|
-
let params;
|
|
244
|
-
if (CLIENT_SIDE_SERIALIZATION_ENABLED) {
|
|
245
|
-
const binaries = buildVectorBinaryParams(queryVector);
|
|
246
|
-
yql = `
|
|
247
|
-
DECLARE $qbin_bit AS String;
|
|
248
|
-
DECLARE $qbinf AS String;
|
|
249
|
-
DECLARE $candidateLimit AS Uint32;
|
|
250
|
-
DECLARE $safeTop AS Uint32;
|
|
251
|
-
DECLARE $uid AS Utf8;
|
|
252
|
-
|
|
253
|
-
$candidates = (
|
|
254
|
-
SELECT point_id
|
|
255
|
-
FROM ${tableName}
|
|
256
|
-
WHERE uid = $uid AND embedding_quantized IS NOT NULL
|
|
257
|
-
ORDER BY ${bitFn}(embedding_quantized, $qbin_bit) ${bitOrder}
|
|
258
|
-
LIMIT $candidateLimit
|
|
259
|
-
);
|
|
260
|
-
|
|
261
|
-
SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
|
|
262
|
-
FROM ${tableName}
|
|
263
|
-
WHERE uid = $uid
|
|
264
|
-
AND point_id IN $candidates
|
|
265
|
-
ORDER BY score ${order}
|
|
266
|
-
LIMIT $safeTop;
|
|
267
|
-
`;
|
|
268
|
-
params = {
|
|
269
|
-
$qbin_bit: typeof TypedValues.bytes === "function"
|
|
270
|
-
? TypedValues.bytes(binaries.bit)
|
|
271
|
-
: binaries.bit,
|
|
272
|
-
$qbinf: typeof TypedValues.bytes === "function"
|
|
273
|
-
? TypedValues.bytes(binaries.float)
|
|
274
|
-
: binaries.float,
|
|
275
|
-
$candidateLimit: TypedValues.uint32(candidateLimit),
|
|
276
|
-
$safeTop: TypedValues.uint32(safeTop),
|
|
277
|
-
$uid: TypedValues.utf8(uid),
|
|
278
|
-
};
|
|
279
|
-
logger.debug({
|
|
280
|
-
tableName,
|
|
281
|
-
distance,
|
|
282
|
-
top,
|
|
283
|
-
safeTop,
|
|
284
|
-
candidateLimit,
|
|
285
|
-
mode: "one_table_approximate_client_side_serialization",
|
|
286
|
-
yql,
|
|
287
|
-
params: {
|
|
288
|
-
uid,
|
|
289
|
-
safeTop,
|
|
290
|
-
candidateLimit,
|
|
291
|
-
vectorLength: queryVector.length,
|
|
292
|
-
vectorPreview: queryVector.slice(0, 3),
|
|
293
|
-
},
|
|
294
|
-
}, "one_table search (approximate): executing YQL with client-side serialization");
|
|
295
|
-
}
|
|
296
|
-
else {
|
|
297
|
-
const qf = buildVectorParam(queryVector);
|
|
298
|
-
yql = `
|
|
299
|
-
DECLARE $qf AS List<Float>;
|
|
300
|
-
DECLARE $candidateLimit AS Uint32;
|
|
301
|
-
DECLARE $safeTop AS Uint32;
|
|
302
|
-
DECLARE $uid AS Utf8;
|
|
303
|
-
|
|
304
|
-
$qbin_bit = Knn::ToBinaryStringBit($qf);
|
|
305
|
-
$qbinf = Knn::ToBinaryStringFloat($qf);
|
|
306
|
-
|
|
307
|
-
$candidates = (
|
|
308
|
-
SELECT point_id
|
|
309
|
-
FROM ${tableName}
|
|
310
|
-
WHERE uid = $uid AND embedding_quantized IS NOT NULL
|
|
311
|
-
ORDER BY ${bitFn}(embedding_quantized, $qbin_bit) ${bitOrder}
|
|
312
|
-
LIMIT $candidateLimit
|
|
313
|
-
);
|
|
314
|
-
|
|
315
|
-
SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
|
|
316
|
-
FROM ${tableName}
|
|
317
|
-
WHERE uid = $uid
|
|
318
|
-
AND point_id IN $candidates
|
|
319
|
-
ORDER BY score ${order}
|
|
320
|
-
LIMIT $safeTop;
|
|
321
|
-
`;
|
|
322
|
-
params = {
|
|
323
|
-
$qf: qf,
|
|
324
|
-
$candidateLimit: TypedValues.uint32(candidateLimit),
|
|
325
|
-
$safeTop: TypedValues.uint32(safeTop),
|
|
326
|
-
$uid: TypedValues.utf8(uid),
|
|
327
|
-
};
|
|
328
|
-
logger.debug({
|
|
329
|
-
tableName,
|
|
330
|
-
distance,
|
|
331
|
-
top,
|
|
332
|
-
safeTop,
|
|
333
|
-
candidateLimit,
|
|
334
|
-
mode: "one_table_approximate",
|
|
335
|
-
yql,
|
|
336
|
-
params: {
|
|
337
|
-
uid,
|
|
338
|
-
safeTop,
|
|
339
|
-
candidateLimit,
|
|
340
|
-
vectorLength: queryVector.length,
|
|
341
|
-
vectorPreview: queryVector.slice(0, 3),
|
|
342
|
-
},
|
|
343
|
-
}, "one_table search (approximate): executing YQL");
|
|
344
|
-
}
|
|
345
|
-
const settings = createExecuteQuerySettingsWithTimeout({
|
|
346
|
-
keepInCache: true,
|
|
347
|
-
idempotent: true,
|
|
348
|
-
timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
|
|
349
|
-
});
|
|
350
|
-
const rs = await s.executeQuery(yql, params, undefined, settings);
|
|
351
|
-
const rowset = rs.resultSets?.[0];
|
|
352
|
-
const rows = (rowset?.rows ?? []);
|
|
353
|
-
return rows.map((row) => {
|
|
354
|
-
const id = row.items?.[0]?.textValue;
|
|
355
|
-
if (typeof id !== "string") {
|
|
356
|
-
throw new Error("point_id is missing in YDB search result");
|
|
357
|
-
}
|
|
358
|
-
let payload;
|
|
359
|
-
let scoreIdx = 1;
|
|
360
|
-
if (withPayload) {
|
|
361
|
-
const payloadText = row.items?.[1]?.textValue;
|
|
362
|
-
if (payloadText) {
|
|
363
|
-
try {
|
|
364
|
-
payload = JSON.parse(payloadText);
|
|
365
|
-
}
|
|
366
|
-
catch {
|
|
367
|
-
payload = undefined;
|
|
368
|
-
}
|
|
369
|
-
}
|
|
370
|
-
scoreIdx = 2;
|
|
371
|
-
}
|
|
372
|
-
const score = Number(row.items?.[scoreIdx]?.floatValue ?? row.items?.[scoreIdx]?.textValue);
|
|
373
|
-
return { id, score, ...(payload ? { payload } : {}) };
|
|
374
|
-
});
|
|
375
|
-
});
|
|
376
|
-
return results;
|
|
377
|
-
}
|
|
378
|
-
export async function searchPointsOneTable(tableName, queryVector, top, withPayload, distance, dimension, uid, mode, overfetchMultiplier) {
|
|
379
|
-
if (mode === SearchMode.Exact) {
|
|
380
|
-
return await searchPointsOneTableExact(tableName, queryVector, top, withPayload, distance, dimension, uid);
|
|
381
|
-
}
|
|
382
|
-
return await searchPointsOneTableApproximate(tableName, queryVector, top, withPayload, distance, dimension, uid, overfetchMultiplier);
|
|
383
|
-
}
|
|
384
|
-
export async function deletePointsOneTable(tableName, ids, uid) {
|
|
385
|
-
let deleted = 0;
|
|
386
|
-
await withSession(async (s) => {
|
|
387
|
-
const settings = createExecuteQuerySettings();
|
|
388
|
-
for (const id of ids) {
|
|
389
|
-
const yql = `
|
|
390
|
-
DECLARE $uid AS Utf8;
|
|
391
|
-
DECLARE $id AS Utf8;
|
|
392
|
-
DELETE FROM ${tableName} WHERE uid = $uid AND point_id = $id;
|
|
393
|
-
`;
|
|
394
|
-
const params = {
|
|
395
|
-
$uid: TypedValues.utf8(uid),
|
|
396
|
-
$id: TypedValues.utf8(String(id)),
|
|
397
|
-
};
|
|
398
|
-
await s.executeQuery(yql, params, undefined, settings);
|
|
399
|
-
deleted += 1;
|
|
400
|
-
}
|
|
401
|
-
});
|
|
402
|
-
return deleted;
|
|
403
|
-
}
|
|
1
|
+
export { searchPointsOneTable } from "./pointsRepo.one-table/Search/index.js";
|
|
2
|
+
export { upsertPointsOneTable } from "./pointsRepo.one-table/Upsert.js";
|
|
3
|
+
export { deletePointsOneTable, deletePointsByPathSegmentsOneTable, } from "./pointsRepo.one-table/Delete.js";
|
|
@@ -3,15 +3,67 @@ import { putCollectionIndex, createCollection, getCollection, deleteCollection,
|
|
|
3
3
|
import { QdrantServiceError } from "../services/errors.js";
|
|
4
4
|
import { logger } from "../logging/logger.js";
|
|
5
5
|
export const collectionsRouter = Router();
|
|
6
|
+
// Placeholder defaults to satisfy Qdrant `CollectionInfo` shape. These values are
|
|
7
|
+
// not used for execution in ydb-qdrant, only for client compatibility.
|
|
8
|
+
const DEFAULT_HNSW_CONFIG = {
|
|
9
|
+
m: 16,
|
|
10
|
+
ef_construct: 100,
|
|
11
|
+
full_scan_threshold: 10000,
|
|
12
|
+
max_indexing_threads: 0,
|
|
13
|
+
on_disk: false,
|
|
14
|
+
};
|
|
15
|
+
const DEFAULT_OPTIMIZERS_CONFIG = {
|
|
16
|
+
deleted_threshold: 0.2,
|
|
17
|
+
vacuum_min_vector_number: 1000,
|
|
18
|
+
default_segment_number: 0,
|
|
19
|
+
indexing_threshold: 10000,
|
|
20
|
+
flush_interval_sec: 5,
|
|
21
|
+
};
|
|
22
|
+
function mapVectorDatatype(dataType) {
|
|
23
|
+
// Our service exposes `float`; Qdrant uses `float32`/`float16`/`uint8`.
|
|
24
|
+
if (dataType === "float16")
|
|
25
|
+
return "float16";
|
|
26
|
+
if (dataType === "uint8")
|
|
27
|
+
return "uint8";
|
|
28
|
+
return "float32";
|
|
29
|
+
}
|
|
30
|
+
function toQdrantCollectionInfo(result) {
|
|
31
|
+
const vectors = result.vectors;
|
|
32
|
+
const datatype = mapVectorDatatype(vectors?.data_type);
|
|
33
|
+
const config = {
|
|
34
|
+
params: {
|
|
35
|
+
vectors: {
|
|
36
|
+
size: vectors.size,
|
|
37
|
+
distance: vectors.distance,
|
|
38
|
+
datatype,
|
|
39
|
+
on_disk: false,
|
|
40
|
+
},
|
|
41
|
+
shard_number: 1,
|
|
42
|
+
replication_factor: 1,
|
|
43
|
+
write_consistency_factor: 1,
|
|
44
|
+
on_disk_payload: false,
|
|
45
|
+
},
|
|
46
|
+
hnsw_config: DEFAULT_HNSW_CONFIG,
|
|
47
|
+
optimizer_config: DEFAULT_OPTIMIZERS_CONFIG,
|
|
48
|
+
};
|
|
49
|
+
return {
|
|
50
|
+
status: "green",
|
|
51
|
+
optimizer_status: "ok",
|
|
52
|
+
segments_count: 1,
|
|
53
|
+
config,
|
|
54
|
+
payload_schema: {},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
6
57
|
collectionsRouter.put("/:collection/index", async (req, res) => {
|
|
7
58
|
try {
|
|
8
|
-
|
|
59
|
+
await putCollectionIndex({
|
|
9
60
|
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
10
61
|
collection: String(req.params.collection),
|
|
11
62
|
apiKey: req.header("api-key") ?? undefined,
|
|
12
63
|
userAgent: req.header("User-Agent") ?? undefined,
|
|
13
64
|
});
|
|
14
|
-
|
|
65
|
+
// Qdrant compatibility: index operations return boolean `result`.
|
|
66
|
+
res.json({ status: "ok", result: true });
|
|
15
67
|
}
|
|
16
68
|
catch (err) {
|
|
17
69
|
if (err instanceof QdrantServiceError) {
|
|
@@ -24,13 +76,14 @@ collectionsRouter.put("/:collection/index", async (req, res) => {
|
|
|
24
76
|
});
|
|
25
77
|
collectionsRouter.put("/:collection", async (req, res) => {
|
|
26
78
|
try {
|
|
27
|
-
|
|
79
|
+
await createCollection({
|
|
28
80
|
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
29
81
|
collection: String(req.params.collection),
|
|
30
82
|
apiKey: req.header("api-key") ?? undefined,
|
|
31
83
|
userAgent: req.header("User-Agent") ?? undefined,
|
|
32
84
|
}, req.body);
|
|
33
|
-
|
|
85
|
+
// Qdrant compatibility: create collection returns boolean `result`.
|
|
86
|
+
res.json({ status: "ok", result: true });
|
|
34
87
|
}
|
|
35
88
|
catch (err) {
|
|
36
89
|
if (err instanceof QdrantServiceError) {
|
|
@@ -49,7 +102,7 @@ collectionsRouter.get("/:collection", async (req, res) => {
|
|
|
49
102
|
apiKey: req.header("api-key") ?? undefined,
|
|
50
103
|
userAgent: req.header("User-Agent") ?? undefined,
|
|
51
104
|
});
|
|
52
|
-
res.json({ status: "ok", result });
|
|
105
|
+
res.json({ status: "ok", result: toQdrantCollectionInfo(result) });
|
|
53
106
|
}
|
|
54
107
|
catch (err) {
|
|
55
108
|
if (err instanceof QdrantServiceError) {
|
|
@@ -62,13 +115,14 @@ collectionsRouter.get("/:collection", async (req, res) => {
|
|
|
62
115
|
});
|
|
63
116
|
collectionsRouter.delete("/:collection", async (req, res) => {
|
|
64
117
|
try {
|
|
65
|
-
|
|
118
|
+
await deleteCollection({
|
|
66
119
|
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
67
120
|
collection: String(req.params.collection),
|
|
68
121
|
apiKey: req.header("api-key") ?? undefined,
|
|
69
122
|
userAgent: req.header("User-Agent") ?? undefined,
|
|
70
123
|
});
|
|
71
|
-
|
|
124
|
+
// Qdrant compatibility: delete collection returns boolean `result`.
|
|
125
|
+
res.json({ status: "ok", result: true });
|
|
72
126
|
}
|
|
73
127
|
catch (err) {
|
|
74
128
|
if (err instanceof QdrantServiceError) {
|
package/dist/routes/points.js
CHANGED
|
@@ -2,9 +2,23 @@ import { Router } from "express";
|
|
|
2
2
|
import { upsertPoints, searchPoints, queryPoints, deletePoints, } from "../services/PointsService.js";
|
|
3
3
|
import { QdrantServiceError } from "../services/errors.js";
|
|
4
4
|
import { logger } from "../logging/logger.js";
|
|
5
|
-
import {
|
|
5
|
+
import { SEARCH_OPERATION_TIMEOUT_MS, UPSERT_OPERATION_TIMEOUT_MS, } from "../config/env.js";
|
|
6
|
+
import { getAbortErrorCause, isCompilationTimeoutError, isTimeoutAbortError, } from "../ydb/client.js";
|
|
6
7
|
import { scheduleExit } from "../utils/exit.js";
|
|
7
8
|
export const pointsRouter = Router();
|
|
9
|
+
function toQdrantScoredPoint(hit) {
|
|
10
|
+
// Qdrant's ScoredPoint includes a mandatory `version`.
|
|
11
|
+
// We don't track versions; emit a stable default.
|
|
12
|
+
return {
|
|
13
|
+
id: hit.id,
|
|
14
|
+
version: 0,
|
|
15
|
+
score: hit.score,
|
|
16
|
+
payload: hit.payload ?? null,
|
|
17
|
+
vector: null,
|
|
18
|
+
shard_key: null,
|
|
19
|
+
order_value: null,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
8
22
|
// Qdrant-compatible: PUT /collections/:collection/points (upsert)
|
|
9
23
|
pointsRouter.put("/:collection/points", async (req, res) => {
|
|
10
24
|
try {
|
|
@@ -27,6 +41,17 @@ pointsRouter.put("/:collection/points", async (req, res) => {
|
|
|
27
41
|
scheduleExit(1);
|
|
28
42
|
return;
|
|
29
43
|
}
|
|
44
|
+
if (isTimeoutAbortError(err)) {
|
|
45
|
+
logger.error({
|
|
46
|
+
err,
|
|
47
|
+
errCause: getAbortErrorCause(err),
|
|
48
|
+
timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
|
|
49
|
+
}, "YDB upsert operation timed out");
|
|
50
|
+
return res.status(500).json({
|
|
51
|
+
status: "error",
|
|
52
|
+
error: `upsert operation timed out after ${UPSERT_OPERATION_TIMEOUT_MS}ms`,
|
|
53
|
+
});
|
|
54
|
+
}
|
|
30
55
|
logger.error({ err }, "upsert points (PUT) failed");
|
|
31
56
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
32
57
|
}
|
|
@@ -52,6 +77,17 @@ pointsRouter.post("/:collection/points/upsert", async (req, res) => {
|
|
|
52
77
|
scheduleExit(1);
|
|
53
78
|
return;
|
|
54
79
|
}
|
|
80
|
+
if (isTimeoutAbortError(err)) {
|
|
81
|
+
logger.error({
|
|
82
|
+
err,
|
|
83
|
+
errCause: getAbortErrorCause(err),
|
|
84
|
+
timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
|
|
85
|
+
}, "YDB upsert operation timed out");
|
|
86
|
+
return res.status(500).json({
|
|
87
|
+
status: "error",
|
|
88
|
+
error: `upsert operation timed out after ${UPSERT_OPERATION_TIMEOUT_MS}ms`,
|
|
89
|
+
});
|
|
90
|
+
}
|
|
55
91
|
logger.error({ err }, "upsert points failed");
|
|
56
92
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
57
93
|
}
|
|
@@ -64,7 +100,12 @@ pointsRouter.post("/:collection/points/search", async (req, res) => {
|
|
|
64
100
|
apiKey: req.header("api-key") ?? undefined,
|
|
65
101
|
userAgent: req.header("User-Agent") ?? undefined,
|
|
66
102
|
}, req.body);
|
|
67
|
-
|
|
103
|
+
// Qdrant compatibility: REST API returns `result` as an array of points.
|
|
104
|
+
// Keep service return shape internal (`{ points: [...] }`).
|
|
105
|
+
res.json({
|
|
106
|
+
status: "ok",
|
|
107
|
+
result: result.points.map(toQdrantScoredPoint),
|
|
108
|
+
});
|
|
68
109
|
}
|
|
69
110
|
catch (err) {
|
|
70
111
|
if (err instanceof QdrantServiceError) {
|
|
@@ -77,6 +118,17 @@ pointsRouter.post("/:collection/points/search", async (req, res) => {
|
|
|
77
118
|
scheduleExit(1);
|
|
78
119
|
return;
|
|
79
120
|
}
|
|
121
|
+
if (isTimeoutAbortError(err)) {
|
|
122
|
+
logger.error({
|
|
123
|
+
err,
|
|
124
|
+
errCause: getAbortErrorCause(err),
|
|
125
|
+
timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
|
|
126
|
+
}, "YDB search operation timed out");
|
|
127
|
+
return res.status(500).json({
|
|
128
|
+
status: "error",
|
|
129
|
+
error: `search operation timed out after ${SEARCH_OPERATION_TIMEOUT_MS}ms`,
|
|
130
|
+
});
|
|
131
|
+
}
|
|
80
132
|
logger.error({ err }, "search points failed");
|
|
81
133
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
82
134
|
}
|
|
@@ -90,7 +142,12 @@ pointsRouter.post("/:collection/points/query", async (req, res) => {
|
|
|
90
142
|
apiKey: req.header("api-key") ?? undefined,
|
|
91
143
|
userAgent: req.header("User-Agent") ?? undefined,
|
|
92
144
|
}, req.body);
|
|
93
|
-
|
|
145
|
+
// Qdrant compatibility: /points/query returns `result` as an object.
|
|
146
|
+
// (Unlike /points/search, where result is a list.)
|
|
147
|
+
const qdrantResult = {
|
|
148
|
+
points: result.points.map(toQdrantScoredPoint),
|
|
149
|
+
};
|
|
150
|
+
res.json({ status: "ok", result: qdrantResult });
|
|
94
151
|
}
|
|
95
152
|
catch (err) {
|
|
96
153
|
if (err instanceof QdrantServiceError) {
|
|
@@ -103,6 +160,17 @@ pointsRouter.post("/:collection/points/query", async (req, res) => {
|
|
|
103
160
|
scheduleExit(1);
|
|
104
161
|
return;
|
|
105
162
|
}
|
|
163
|
+
if (isTimeoutAbortError(err)) {
|
|
164
|
+
logger.error({
|
|
165
|
+
err,
|
|
166
|
+
errCause: getAbortErrorCause(err),
|
|
167
|
+
timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
|
|
168
|
+
}, "YDB search operation timed out");
|
|
169
|
+
return res.status(500).json({
|
|
170
|
+
status: "error",
|
|
171
|
+
error: `search operation timed out after ${SEARCH_OPERATION_TIMEOUT_MS}ms`,
|
|
172
|
+
});
|
|
173
|
+
}
|
|
106
174
|
logger.error({ err }, "search points (query) failed");
|
|
107
175
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
108
176
|
}
|
package/dist/server.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
import type { Request, Response } from "express";
|
|
2
2
|
export declare function healthHandler(_req: Request, res: Response): Promise<void>;
|
|
3
|
+
export declare function rootHandler(_req: Request, res: Response): void;
|
|
3
4
|
export declare function buildServer(): import("express-serve-static-core").Express;
|