ydb-qdrant 8.1.0 → 9.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -18
- package/dist/SmokeTest.js +2 -2
- package/dist/compute/ComputePool.d.ts +5 -0
- package/dist/compute/ComputePool.js +64 -0
- package/dist/compute/ComputeWorker.d.ts +36 -0
- package/dist/compute/ComputeWorker.js +84 -0
- package/dist/config/env.d.ts +24 -7
- package/dist/config/env.js +65 -35
- package/dist/index.d.ts +2 -0
- package/dist/index.js +92 -2
- package/dist/logging/DeployLogFormatter.d.ts +2 -0
- package/dist/logging/DeployLogFormatter.js +131 -0
- package/dist/logging/logger.js +13 -1
- package/dist/logging/requestContext.d.ts +17 -0
- package/dist/logging/requestContext.js +43 -0
- package/dist/middleware/requestLogger.js +134 -6
- package/dist/middleware/upsertBodyPhase.d.ts +6 -0
- package/dist/middleware/upsertBodyPhase.js +184 -0
- package/dist/middleware/upsertRequestTimeout.d.ts +16 -0
- package/dist/middleware/upsertRequestTimeout.js +158 -0
- package/dist/package/api.d.ts +20 -12
- package/dist/package/api.js +57 -28
- package/dist/qdrant/QdrantRestTypes.d.ts +4 -0
- package/dist/qdrant/Requests.d.ts +97 -0
- package/dist/qdrant/Requests.js +72 -0
- package/dist/repositories/collectionsRepo.d.ts +18 -2
- package/dist/repositories/collectionsRepo.js +103 -7
- package/dist/repositories/collectionsRepo.one-table.d.ts +4 -3
- package/dist/repositories/collectionsRepo.one-table.js +99 -36
- package/dist/repositories/collectionsRepo.shared.d.ts +2 -2
- package/dist/repositories/collectionsRepo.shared.js +9 -4
- package/dist/repositories/pointsRepo.d.ts +6 -4
- package/dist/repositories/pointsRepo.js +8 -7
- package/dist/repositories/pointsRepo.one-table/Delete.d.ts +2 -2
- package/dist/repositories/pointsRepo.one-table/Delete.js +157 -60
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.d.ts +7 -5
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.js +44 -13
- package/dist/repositories/pointsRepo.one-table/Retrieve.d.ts +6 -0
- package/dist/repositories/pointsRepo.one-table/Retrieve.js +69 -0
- package/dist/repositories/pointsRepo.one-table/Search.d.ts +2 -3
- package/dist/repositories/pointsRepo.one-table/Search.js +102 -124
- package/dist/repositories/pointsRepo.one-table/Upsert.d.ts +2 -2
- package/dist/repositories/pointsRepo.one-table/Upsert.js +244 -48
- package/dist/repositories/pointsRepo.one-table.d.ts +1 -0
- package/dist/repositories/pointsRepo.one-table.js +1 -0
- package/dist/routes/collections.js +45 -36
- package/dist/routes/points.js +145 -56
- package/dist/server.js +42 -6
- package/dist/services/CollectionService.d.ts +7 -5
- package/dist/services/CollectionService.js +12 -9
- package/dist/services/CollectionService.one-table.js +1 -2
- package/dist/services/CollectionService.shared.d.ts +6 -5
- package/dist/services/CollectionService.shared.js +28 -12
- package/dist/services/PointsService.d.ts +8 -0
- package/dist/services/PointsService.js +132 -15
- package/dist/types.d.ts +4 -94
- package/dist/types.js +1 -54
- package/dist/utils/EnvParsers.d.ts +5 -0
- package/dist/utils/EnvParsers.js +30 -0
- package/dist/utils/PayloadSign.d.ts +4 -0
- package/dist/utils/PayloadSign.js +18 -0
- package/dist/utils/distance.d.ts +1 -12
- package/dist/utils/distance.js +0 -21
- package/dist/utils/pathPrefix.d.ts +3 -0
- package/dist/utils/pathPrefix.js +47 -0
- package/dist/utils/prefixExpansion.d.ts +1 -0
- package/dist/utils/prefixExpansion.js +11 -0
- package/dist/utils/qdrantResponse.d.ts +13 -0
- package/dist/utils/qdrantResponse.js +12 -0
- package/dist/utils/requestIdentity.d.ts +8 -0
- package/dist/utils/requestIdentity.js +52 -0
- package/dist/utils/retry.d.ts +2 -0
- package/dist/utils/retry.js +55 -11
- package/dist/utils/tenant.d.ts +12 -6
- package/dist/utils/tenant.js +41 -32
- package/dist/utils/vectorBinary.d.ts +0 -1
- package/dist/utils/vectorBinary.js +0 -98
- package/dist/utils/ydbErrors.d.ts +1 -0
- package/dist/utils/ydbErrors.js +14 -0
- package/dist/ydb/bootstrapMetaTable.js +14 -2
- package/dist/ydb/client.d.ts +10 -2
- package/dist/ydb/client.js +83 -24
- package/dist/ydb/helpers.d.ts +0 -1
- package/dist/ydb/helpers.js +1 -2
- package/dist/ydb/schema.d.ts +2 -0
- package/dist/ydb/schema.js +84 -7
- package/package.json +10 -5
|
@@ -1,12 +1,22 @@
|
|
|
1
|
-
import { UpsertPointsReq, SearchReq, DeletePointsReq } from "../
|
|
1
|
+
import { UpsertPointsReq, SearchReq, DeletePointsReq, RetrievePointsReq, } from "../qdrant/Requests.js";
|
|
2
2
|
import { ensureMetaTable } from "../ydb/schema.js";
|
|
3
|
-
import { getCollectionMeta, touchCollectionLastAccess, } from "../repositories/collectionsRepo.js";
|
|
4
|
-
import { deletePoints as repoDeletePoints, deletePointsByPathSegments as repoDeletePointsByPathSegments, searchPoints as repoSearchPoints, upsertPoints as repoUpsertPoints, } from "../repositories/pointsRepo.js";
|
|
3
|
+
import { getCollectionMeta, touchCollectionLastAccess, deleteAllPointsForCollection, } from "../repositories/collectionsRepo.js";
|
|
4
|
+
import { deletePoints as repoDeletePoints, deletePointsByPathSegments as repoDeletePointsByPathSegments, searchPoints as repoSearchPoints, upsertPoints as repoUpsertPoints, retrievePointsByIds as repoRetrievePointsByIds, } from "../repositories/pointsRepo.js";
|
|
5
5
|
import { logger } from "../logging/logger.js";
|
|
6
|
+
import { elapsedMsSince, getElapsedMsSinceRequestStart, getMonotonicTimeNs, } from "../logging/requestContext.js";
|
|
6
7
|
import { QdrantServiceError, isVectorDimensionMismatchError, } from "./errors.js";
|
|
7
8
|
import { normalizeCollectionContextShared } from "./CollectionService.shared.js";
|
|
8
9
|
import { resolvePointsTableAndUidOneTable } from "./CollectionService.one-table.js";
|
|
9
10
|
import { normalizeSearchBodyForSearch, normalizeSearchBodyForQuery, } from "../utils/normalization.js";
|
|
11
|
+
import { normalizePathSegments } from "../utils/pathPrefix.js";
|
|
12
|
+
const MAX_LOGGED_UPSERT_PATHS = 20;
|
|
13
|
+
const MAX_LOGGED_DELETE_PATHS = 20;
|
|
14
|
+
function isEmptyFilter(filter) {
|
|
15
|
+
if (typeof filter !== "object" || filter === null)
|
|
16
|
+
return false;
|
|
17
|
+
const f = filter;
|
|
18
|
+
return Array.isArray(f.must) && f.must.length === 0;
|
|
19
|
+
}
|
|
10
20
|
function parsePathSegmentsFilterToPaths(filter) {
|
|
11
21
|
const extractMust = (must) => {
|
|
12
22
|
if (!Array.isArray(must) || must.length === 0)
|
|
@@ -63,35 +73,90 @@ function parsePathSegmentsFilterToPaths(filter) {
|
|
|
63
73
|
}
|
|
64
74
|
return null;
|
|
65
75
|
}
|
|
76
|
+
function collectPathLogFields(pathStrings, maxLoggedPaths) {
|
|
77
|
+
const uniquePaths = new Set();
|
|
78
|
+
const paths = [];
|
|
79
|
+
for (const path of pathStrings) {
|
|
80
|
+
if (uniquePaths.has(path)) {
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
uniquePaths.add(path);
|
|
84
|
+
if (paths.length < maxLoggedPaths) {
|
|
85
|
+
paths.push(path);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return {
|
|
89
|
+
pathCount: uniquePaths.size,
|
|
90
|
+
paths,
|
|
91
|
+
pathsTruncated: uniquePaths.size > paths.length,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
function collectUpsertPathLogFields(points) {
|
|
95
|
+
const pathStrings = [];
|
|
96
|
+
for (const point of points) {
|
|
97
|
+
const pathSegments = normalizePathSegments(point.payload?.pathSegments);
|
|
98
|
+
if (!pathSegments) {
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
pathStrings.push(pathSegments.join("/"));
|
|
102
|
+
}
|
|
103
|
+
return collectPathLogFields(pathStrings, MAX_LOGGED_UPSERT_PATHS);
|
|
104
|
+
}
|
|
66
105
|
export async function upsertPoints(ctx, body) {
|
|
106
|
+
const serviceStartNs = getMonotonicTimeNs();
|
|
107
|
+
const metaLookupStartNs = getMonotonicTimeNs();
|
|
67
108
|
await ensureMetaTable();
|
|
68
|
-
const normalized = normalizeCollectionContextShared(ctx.
|
|
109
|
+
const normalized = normalizeCollectionContextShared(ctx.userUid, ctx.collection, ctx.userAgent);
|
|
69
110
|
const meta = await getCollectionMeta(normalized.metaKey);
|
|
111
|
+
const metaLookupMs = elapsedMsSince(metaLookupStartNs);
|
|
70
112
|
if (!meta) {
|
|
71
113
|
throw new QdrantServiceError(404, {
|
|
72
114
|
status: "error",
|
|
73
115
|
error: "collection not found",
|
|
74
116
|
});
|
|
75
117
|
}
|
|
118
|
+
const requestValidationStartNs = getMonotonicTimeNs();
|
|
76
119
|
const parsed = UpsertPointsReq.safeParse(body);
|
|
120
|
+
const requestValidationMs = elapsedMsSince(requestValidationStartNs);
|
|
77
121
|
if (!parsed.success) {
|
|
78
122
|
throw new QdrantServiceError(400, {
|
|
79
123
|
status: "error",
|
|
80
124
|
error: parsed.error.flatten(),
|
|
81
125
|
});
|
|
82
126
|
}
|
|
127
|
+
const uidResolveStartNs = getMonotonicTimeNs();
|
|
83
128
|
const { tableName, uid } = await resolvePointsTableAndUidOneTable(normalized);
|
|
129
|
+
const uidResolveMs = elapsedMsSince(uidResolveStartNs);
|
|
130
|
+
const upsertPathLogFields = collectUpsertPathLogFields(parsed.data.points);
|
|
131
|
+
logger.info({
|
|
132
|
+
collection: normalized.collection,
|
|
133
|
+
uid,
|
|
134
|
+
pointCount: parsed.data.points.length,
|
|
135
|
+
...upsertPathLogFields,
|
|
136
|
+
}, "upsertPoints: payload paths");
|
|
137
|
+
logger.info({
|
|
138
|
+
phase: "upsertPrepare",
|
|
139
|
+
collection: normalized.collection,
|
|
140
|
+
uid,
|
|
141
|
+
metaLookupMs,
|
|
142
|
+
requestValidationMs,
|
|
143
|
+
uidResolveMs,
|
|
144
|
+
pointCount: parsed.data.points.length,
|
|
145
|
+
pathCount: upsertPathLogFields.pathCount,
|
|
146
|
+
timeToHandlerMs: getElapsedMsSinceRequestStart(),
|
|
147
|
+
}, "upsert: prepare phase");
|
|
148
|
+
const repoStartNs = getMonotonicTimeNs();
|
|
84
149
|
let upserted;
|
|
85
150
|
try {
|
|
86
|
-
upserted = await repoUpsertPoints(tableName, parsed.data.points, meta.dimension, uid);
|
|
151
|
+
upserted = await repoUpsertPoints(tableName, parsed.data.points, meta.dimension, uid, ctx.apiKey);
|
|
87
152
|
}
|
|
88
153
|
catch (err) {
|
|
89
154
|
if (isVectorDimensionMismatchError(err)) {
|
|
90
155
|
logger.warn({
|
|
91
|
-
tenant: normalized.tenant,
|
|
92
156
|
collection: normalized.collection,
|
|
93
157
|
table: tableName,
|
|
94
158
|
dimension: meta.dimension,
|
|
159
|
+
uid,
|
|
95
160
|
}, "upsertPoints: vector dimension mismatch");
|
|
96
161
|
throw new QdrantServiceError(400, {
|
|
97
162
|
status: "error",
|
|
@@ -100,17 +165,28 @@ export async function upsertPoints(ctx, body) {
|
|
|
100
165
|
}
|
|
101
166
|
throw err;
|
|
102
167
|
}
|
|
168
|
+
const repoTotalMs = elapsedMsSince(repoStartNs);
|
|
169
|
+
const lastAccessStartNs = getMonotonicTimeNs();
|
|
103
170
|
await touchCollectionLastAccess(normalized.metaKey);
|
|
171
|
+
const lastAccessMs = elapsedMsSince(lastAccessStartNs);
|
|
172
|
+
logger.info({
|
|
173
|
+
phase: "upsertComplete",
|
|
174
|
+
collection: normalized.collection,
|
|
175
|
+
uid,
|
|
176
|
+
serviceTotalMs: elapsedMsSince(serviceStartNs),
|
|
177
|
+
repoTotalMs,
|
|
178
|
+
lastAccessMs,
|
|
179
|
+
pointCount: parsed.data.points.length,
|
|
180
|
+
}, "upsert: completed");
|
|
104
181
|
return { upserted };
|
|
105
182
|
}
|
|
106
183
|
async function executeSearch(ctx, normalizedSearch, source) {
|
|
107
184
|
await ensureMetaTable();
|
|
108
|
-
const normalized = normalizeCollectionContextShared(ctx.
|
|
109
|
-
logger.info({
|
|
185
|
+
const normalized = normalizeCollectionContextShared(ctx.userUid, ctx.collection, ctx.userAgent);
|
|
186
|
+
logger.info({ collection: normalized.collection }, `${source}: resolve collection meta`);
|
|
110
187
|
const meta = await getCollectionMeta(normalized.metaKey);
|
|
111
188
|
if (!meta) {
|
|
112
189
|
logger.warn({
|
|
113
|
-
tenant: normalized.tenant,
|
|
114
190
|
collection: normalized.collection,
|
|
115
191
|
metaKey: normalized.metaKey,
|
|
116
192
|
}, `${source}: collection not found`);
|
|
@@ -126,7 +202,6 @@ async function executeSearch(ctx, normalizedSearch, source) {
|
|
|
126
202
|
});
|
|
127
203
|
if (!parsed.success) {
|
|
128
204
|
logger.warn({
|
|
129
|
-
tenant: normalized.tenant,
|
|
130
205
|
collection: normalized.collection,
|
|
131
206
|
issues: parsed.error.issues,
|
|
132
207
|
}, `${source}: invalid payload`);
|
|
@@ -137,8 +212,8 @@ async function executeSearch(ctx, normalizedSearch, source) {
|
|
|
137
212
|
}
|
|
138
213
|
const { tableName, uid } = await resolvePointsTableAndUidOneTable(normalized);
|
|
139
214
|
logger.info({
|
|
140
|
-
tenant: normalized.tenant,
|
|
141
215
|
collection: normalized.collection,
|
|
216
|
+
uid,
|
|
142
217
|
top: parsed.data.top,
|
|
143
218
|
queryVectorLen: parsed.data.vector.length,
|
|
144
219
|
collectionDim: meta.dimension,
|
|
@@ -148,16 +223,16 @@ async function executeSearch(ctx, normalizedSearch, source) {
|
|
|
148
223
|
const filterPaths = parsePathSegmentsFilterToPaths(normalizedSearch.filter);
|
|
149
224
|
let hits;
|
|
150
225
|
try {
|
|
151
|
-
hits = await repoSearchPoints(tableName, parsed.data.vector, parsed.data.top, parsed.data.with_payload, meta.distance, meta.dimension, uid, filterPaths ?? undefined);
|
|
226
|
+
hits = await repoSearchPoints(tableName, parsed.data.vector, parsed.data.top, parsed.data.with_payload, meta.distance, meta.dimension, uid, ctx.apiKey, filterPaths ?? undefined);
|
|
152
227
|
}
|
|
153
228
|
catch (err) {
|
|
154
229
|
if (isVectorDimensionMismatchError(err)) {
|
|
155
230
|
logger.warn({
|
|
156
|
-
tenant: normalized.tenant,
|
|
157
231
|
collection: normalized.collection,
|
|
158
232
|
table: tableName,
|
|
159
233
|
dimension: meta.dimension,
|
|
160
234
|
queryVectorLen: parsed.data.vector.length,
|
|
235
|
+
uid,
|
|
161
236
|
}, `${source}: vector dimension mismatch`);
|
|
162
237
|
throw new QdrantServiceError(400, {
|
|
163
238
|
status: "error",
|
|
@@ -188,7 +263,6 @@ async function executeSearch(ctx, normalizedSearch, source) {
|
|
|
188
263
|
return hit.score <= threshold;
|
|
189
264
|
});
|
|
190
265
|
logger.info({
|
|
191
|
-
tenant: normalized.tenant,
|
|
192
266
|
collection: normalized.collection,
|
|
193
267
|
hits: hits.length,
|
|
194
268
|
}, `${source}: completed`);
|
|
@@ -202,9 +276,35 @@ export async function queryPoints(ctx, body) {
|
|
|
202
276
|
const normalizedSearch = normalizeSearchBodyForQuery(body);
|
|
203
277
|
return await executeSearch(ctx, normalizedSearch, "query");
|
|
204
278
|
}
|
|
279
|
+
export async function retrievePoints(ctx, body) {
|
|
280
|
+
await ensureMetaTable();
|
|
281
|
+
const normalized = normalizeCollectionContextShared(ctx.userUid, ctx.collection, ctx.userAgent);
|
|
282
|
+
const meta = await getCollectionMeta(normalized.metaKey);
|
|
283
|
+
if (!meta) {
|
|
284
|
+
throw new QdrantServiceError(404, {
|
|
285
|
+
status: "error",
|
|
286
|
+
error: "collection not found",
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
const parsed = RetrievePointsReq.safeParse(body);
|
|
290
|
+
if (!parsed.success) {
|
|
291
|
+
throw new QdrantServiceError(400, {
|
|
292
|
+
status: "error",
|
|
293
|
+
error: parsed.error.flatten(),
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
const { tableName, uid } = await resolvePointsTableAndUidOneTable(normalized);
|
|
297
|
+
const points = await repoRetrievePointsByIds(tableName, parsed.data.ids, uid, ctx.apiKey, parsed.data.with_payload);
|
|
298
|
+
await touchCollectionLastAccess(normalized.metaKey);
|
|
299
|
+
return { points };
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* @returns `deleted` — number of points removed, or `-1` when the exact count
|
|
303
|
+
* is unavailable (e.g. bulk BATCH DELETE or clear-all-points).
|
|
304
|
+
*/
|
|
205
305
|
export async function deletePoints(ctx, body) {
|
|
206
306
|
await ensureMetaTable();
|
|
207
|
-
const normalized = normalizeCollectionContextShared(ctx.
|
|
307
|
+
const normalized = normalizeCollectionContextShared(ctx.userUid, ctx.collection, ctx.userAgent);
|
|
208
308
|
const meta = await getCollectionMeta(normalized.metaKey);
|
|
209
309
|
if (!meta) {
|
|
210
310
|
throw new QdrantServiceError(404, {
|
|
@@ -222,8 +322,18 @@ export async function deletePoints(ctx, body) {
|
|
|
222
322
|
const { tableName, uid } = await resolvePointsTableAndUidOneTable(normalized);
|
|
223
323
|
let deleted;
|
|
224
324
|
if ("points" in parsed.data) {
|
|
325
|
+
logger.info({
|
|
326
|
+
collection: normalized.collection,
|
|
327
|
+
uid,
|
|
328
|
+
idCount: parsed.data.points.length,
|
|
329
|
+
}, "deletePoints: by IDs");
|
|
225
330
|
deleted = await repoDeletePoints(tableName, parsed.data.points, uid);
|
|
226
331
|
}
|
|
332
|
+
else if (isEmptyFilter(parsed.data.filter)) {
|
|
333
|
+
logger.info({ collection: normalized.collection, uid }, "deletePoints: clear all points");
|
|
334
|
+
await deleteAllPointsForCollection(uid);
|
|
335
|
+
deleted = -1;
|
|
336
|
+
}
|
|
227
337
|
else {
|
|
228
338
|
const paths = parsePathSegmentsFilterToPaths(parsed.data.filter);
|
|
229
339
|
if (!paths) {
|
|
@@ -232,8 +342,15 @@ export async function deletePoints(ctx, body) {
|
|
|
232
342
|
error: "unsupported delete filter: only pathSegments.N match filters with must/should are supported",
|
|
233
343
|
});
|
|
234
344
|
}
|
|
345
|
+
const deletePathLogFields = collectPathLogFields(paths.map((segments) => segments.join("/")), MAX_LOGGED_DELETE_PATHS);
|
|
346
|
+
logger.info({
|
|
347
|
+
collection: normalized.collection,
|
|
348
|
+
uid,
|
|
349
|
+
...deletePathLogFields,
|
|
350
|
+
}, "deletePoints: by pathSegments filter");
|
|
235
351
|
deleted = await repoDeletePointsByPathSegments(tableName, uid, paths);
|
|
236
352
|
}
|
|
353
|
+
logger.info({ collection: normalized.collection, uid, deleted }, "deletePoints: completed");
|
|
237
354
|
await touchCollectionLastAccess(normalized.metaKey);
|
|
238
355
|
return { deleted };
|
|
239
356
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,94 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
export type
|
|
4
|
-
export type
|
|
5
|
-
export type Payload = QdrantPayload;
|
|
6
|
-
export type WithPayload = QdrantWithPayloadInterface;
|
|
7
|
-
/**
|
|
8
|
-
* Collection metadata from qdr__collections table.
|
|
9
|
-
*
|
|
10
|
-
* @property lastAccessedAt - Timestamp of last access; undefined for collections
|
|
11
|
-
* created before this feature, null if explicitly unset.
|
|
12
|
-
*/
|
|
13
|
-
export interface CollectionMeta {
|
|
14
|
-
table: string;
|
|
15
|
-
dimension: number;
|
|
16
|
-
distance: DistanceKind;
|
|
17
|
-
vectorType: VectorType;
|
|
18
|
-
lastAccessedAt?: Date | null;
|
|
19
|
-
}
|
|
20
|
-
export declare const CreateCollectionReq: z.ZodObject<{
|
|
21
|
-
vectors: z.ZodObject<{
|
|
22
|
-
size: z.ZodNumber;
|
|
23
|
-
distance: z.ZodType<DistanceKind>;
|
|
24
|
-
data_type: z.ZodOptional<z.ZodEnum<{
|
|
25
|
-
float: "float";
|
|
26
|
-
}>>;
|
|
27
|
-
}, z.core.$strip>;
|
|
28
|
-
}, z.core.$strip>;
|
|
29
|
-
export declare const UpsertPointsReq: z.ZodObject<{
|
|
30
|
-
points: z.ZodArray<z.ZodObject<{
|
|
31
|
-
id: z.ZodType<YdbQdrantPointId>;
|
|
32
|
-
vector: z.ZodArray<z.ZodNumber>;
|
|
33
|
-
payload: z.ZodType<Payload | undefined>;
|
|
34
|
-
}, z.core.$strip>>;
|
|
35
|
-
}, z.core.$strip>;
|
|
36
|
-
export type UpsertPoint = YdbQdrantUpsertPoint;
|
|
37
|
-
export type UpsertPointsBody = {
|
|
38
|
-
points: UpsertPoint[];
|
|
39
|
-
};
|
|
40
|
-
export declare const SearchReq: z.ZodObject<{
|
|
41
|
-
vector: z.ZodArray<z.ZodNumber>;
|
|
42
|
-
top: z.ZodNumber;
|
|
43
|
-
with_payload: z.ZodOptional<z.ZodBoolean>;
|
|
44
|
-
}, z.core.$strip>;
|
|
45
|
-
export type SearchPointsBody = {
|
|
46
|
-
vector: number[];
|
|
47
|
-
top?: number;
|
|
48
|
-
limit?: number;
|
|
49
|
-
with_payload?: WithPayload;
|
|
50
|
-
score_threshold?: number | null;
|
|
51
|
-
};
|
|
52
|
-
export declare const DeletePointsByIdsReq: z.ZodObject<{
|
|
53
|
-
points: z.ZodArray<z.ZodType<YdbQdrantPointId, unknown, z.core.$ZodTypeInternals<YdbQdrantPointId, unknown>>>;
|
|
54
|
-
}, z.core.$strip>;
|
|
55
|
-
export declare const DeletePointsByFilterReq: z.ZodObject<{
|
|
56
|
-
filter: z.ZodUnion<readonly [z.ZodObject<{
|
|
57
|
-
must: z.ZodArray<z.ZodObject<{
|
|
58
|
-
key: z.ZodString;
|
|
59
|
-
match: z.ZodObject<{
|
|
60
|
-
value: z.ZodString;
|
|
61
|
-
}, z.core.$strip>;
|
|
62
|
-
}, z.core.$strip>>;
|
|
63
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
64
|
-
should: z.ZodArray<z.ZodObject<{
|
|
65
|
-
must: z.ZodArray<z.ZodObject<{
|
|
66
|
-
key: z.ZodString;
|
|
67
|
-
match: z.ZodObject<{
|
|
68
|
-
value: z.ZodString;
|
|
69
|
-
}, z.core.$strip>;
|
|
70
|
-
}, z.core.$strip>>;
|
|
71
|
-
}, z.core.$strip>>;
|
|
72
|
-
}, z.core.$strip>]>;
|
|
73
|
-
}, z.core.$strip>;
|
|
74
|
-
export declare const DeletePointsReq: z.ZodUnion<readonly [z.ZodObject<{
|
|
75
|
-
points: z.ZodArray<z.ZodType<YdbQdrantPointId, unknown, z.core.$ZodTypeInternals<YdbQdrantPointId, unknown>>>;
|
|
76
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
77
|
-
filter: z.ZodUnion<readonly [z.ZodObject<{
|
|
78
|
-
must: z.ZodArray<z.ZodObject<{
|
|
79
|
-
key: z.ZodString;
|
|
80
|
-
match: z.ZodObject<{
|
|
81
|
-
value: z.ZodString;
|
|
82
|
-
}, z.core.$strip>;
|
|
83
|
-
}, z.core.$strip>>;
|
|
84
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
85
|
-
should: z.ZodArray<z.ZodObject<{
|
|
86
|
-
must: z.ZodArray<z.ZodObject<{
|
|
87
|
-
key: z.ZodString;
|
|
88
|
-
match: z.ZodObject<{
|
|
89
|
-
value: z.ZodString;
|
|
90
|
-
}, z.core.$strip>;
|
|
91
|
-
}, z.core.$strip>>;
|
|
92
|
-
}, z.core.$strip>>;
|
|
93
|
-
}, z.core.$strip>]>;
|
|
94
|
-
}, z.core.$strip>]>;
|
|
1
|
+
export type { DistanceKind, Payload, VectorType, WithPayload, YdbQdrantPointId, YdbQdrantUpsertPoint, } from "./qdrant/QdrantRestTypes.js";
|
|
2
|
+
export { CreateCollectionReq, DeletePointsByFilterReq, DeletePointsByIdsReq, DeletePointsReq, RetrievePointsReq, SearchReq, UpsertPointsReq, } from "./qdrant/Requests.js";
|
|
3
|
+
export type { SearchPointsBody, UpsertPoint, UpsertPointsBody, } from "./qdrant/Requests.js";
|
|
4
|
+
export type { CollectionMeta } from "./repositories/collectionsRepo.js";
|
package/dist/types.js
CHANGED
|
@@ -1,54 +1 @@
|
|
|
1
|
-
|
|
2
|
-
export const CreateCollectionReq = z.object({
|
|
3
|
-
vectors: z.object({
|
|
4
|
-
size: z.number().int().positive(),
|
|
5
|
-
distance: z.enum([
|
|
6
|
-
"Cosine",
|
|
7
|
-
"Euclid",
|
|
8
|
-
"Dot",
|
|
9
|
-
"Manhattan",
|
|
10
|
-
]),
|
|
11
|
-
data_type: z.enum(["float"]).optional(),
|
|
12
|
-
}),
|
|
13
|
-
});
|
|
14
|
-
export const UpsertPointsReq = z.object({
|
|
15
|
-
points: z
|
|
16
|
-
.array(z.object({
|
|
17
|
-
id: z.union([z.string(), z.number()]),
|
|
18
|
-
vector: z.array(z.number()),
|
|
19
|
-
payload: z.record(z.string(), z.unknown()).optional(),
|
|
20
|
-
}))
|
|
21
|
-
.min(1),
|
|
22
|
-
});
|
|
23
|
-
export const SearchReq = z.object({
|
|
24
|
-
vector: z.array(z.number()).min(1),
|
|
25
|
-
top: z.number().int().positive().max(1000),
|
|
26
|
-
with_payload: z.boolean().optional(),
|
|
27
|
-
});
|
|
28
|
-
export const DeletePointsByIdsReq = z.object({
|
|
29
|
-
points: z
|
|
30
|
-
.array(z.union([z.string(), z.number()]))
|
|
31
|
-
.min(1),
|
|
32
|
-
});
|
|
33
|
-
const DeletePointsFilterCondition = z.object({
|
|
34
|
-
key: z.string(),
|
|
35
|
-
match: z.object({
|
|
36
|
-
value: z.string(),
|
|
37
|
-
}),
|
|
38
|
-
});
|
|
39
|
-
const DeletePointsFilterMust = z.object({
|
|
40
|
-
must: z.array(DeletePointsFilterCondition).min(1),
|
|
41
|
-
});
|
|
42
|
-
const DeletePointsFilter = z.union([
|
|
43
|
-
DeletePointsFilterMust,
|
|
44
|
-
z.object({
|
|
45
|
-
should: z.array(DeletePointsFilterMust).min(1),
|
|
46
|
-
}),
|
|
47
|
-
]);
|
|
48
|
-
export const DeletePointsByFilterReq = z.object({
|
|
49
|
-
filter: DeletePointsFilter,
|
|
50
|
-
});
|
|
51
|
-
export const DeletePointsReq = z.union([
|
|
52
|
-
DeletePointsByIdsReq,
|
|
53
|
-
DeletePointsByFilterReq,
|
|
54
|
-
]);
|
|
1
|
+
export { CreateCollectionReq, DeletePointsByFilterReq, DeletePointsByIdsReq, DeletePointsReq, RetrievePointsReq, SearchReq, UpsertPointsReq, } from "./qdrant/Requests.js";
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export function parseBooleanEnv(value, defaultValue) {
|
|
2
|
+
if (value === undefined) {
|
|
3
|
+
return defaultValue;
|
|
4
|
+
}
|
|
5
|
+
const normalized = value.trim().toLowerCase();
|
|
6
|
+
if (normalized === "true" || normalized === "1" || normalized === "yes") {
|
|
7
|
+
return true;
|
|
8
|
+
}
|
|
9
|
+
if (normalized === "false" || normalized === "0" || normalized === "no") {
|
|
10
|
+
return false;
|
|
11
|
+
}
|
|
12
|
+
return defaultValue;
|
|
13
|
+
}
|
|
14
|
+
export function parseIntegerEnv(value, defaultValue, opts) {
|
|
15
|
+
if (value === undefined) {
|
|
16
|
+
return defaultValue;
|
|
17
|
+
}
|
|
18
|
+
const parsed = Number.parseInt(value.trim(), 10);
|
|
19
|
+
if (!Number.isFinite(parsed)) {
|
|
20
|
+
return defaultValue;
|
|
21
|
+
}
|
|
22
|
+
let result = parsed;
|
|
23
|
+
if (opts?.min !== undefined && result < opts.min) {
|
|
24
|
+
result = opts.min;
|
|
25
|
+
}
|
|
26
|
+
if (opts?.max !== undefined && result > opts.max) {
|
|
27
|
+
result = opts.max;
|
|
28
|
+
}
|
|
29
|
+
return result;
|
|
30
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import crypto from "crypto";
|
|
2
|
+
import stableStringify from "fast-json-stable-stringify";
|
|
3
|
+
function canonicalizePayload(payload) {
|
|
4
|
+
const canonical = stableStringify(payload);
|
|
5
|
+
if (typeof canonical !== "string") {
|
|
6
|
+
// JSON.stringify returns undefined for top-level undefined/functions/symbols.
|
|
7
|
+
throw new Error("computePayloadSign: payload is not JSON-serializable");
|
|
8
|
+
}
|
|
9
|
+
return canonical;
|
|
10
|
+
}
|
|
11
|
+
export function computePayloadSign(params) {
|
|
12
|
+
const apiKey = params.apiKey.trim();
|
|
13
|
+
if (apiKey.length === 0) {
|
|
14
|
+
throw new Error("computePayloadSign: apiKey is empty");
|
|
15
|
+
}
|
|
16
|
+
const canonical = canonicalizePayload(params.payload);
|
|
17
|
+
return crypto.createHmac("sha256", apiKey).update(canonical).digest("hex");
|
|
18
|
+
}
|
package/dist/utils/distance.d.ts
CHANGED
|
@@ -1,16 +1,5 @@
|
|
|
1
|
-
import type { DistanceKind } from "../
|
|
1
|
+
import type { DistanceKind } from "../qdrant/QdrantRestTypes.js";
|
|
2
2
|
export declare function mapDistanceToKnnFn(distance: DistanceKind): {
|
|
3
3
|
fn: string;
|
|
4
4
|
order: "ASC" | "DESC";
|
|
5
5
|
};
|
|
6
|
-
/**
|
|
7
|
-
* Maps a user-specified distance metric to a YDB Knn function
|
|
8
|
-
* suitable for bit-quantized vectors (Phase 1 approximate candidate selection).
|
|
9
|
-
* Cosine uses similarity (DESC); other metrics use distance (ASC).
|
|
10
|
-
* For Dot, falls back to CosineDistance as a proxy since there is no
|
|
11
|
-
* direct distance equivalent for inner product.
|
|
12
|
-
*/
|
|
13
|
-
export declare function mapDistanceToBitKnnFn(distance: DistanceKind): {
|
|
14
|
-
fn: string;
|
|
15
|
-
order: "ASC" | "DESC";
|
|
16
|
-
};
|
package/dist/utils/distance.js
CHANGED
|
@@ -12,24 +12,3 @@ export function mapDistanceToKnnFn(distance) {
|
|
|
12
12
|
return { fn: "Knn::CosineDistance", order: "ASC" };
|
|
13
13
|
}
|
|
14
14
|
}
|
|
15
|
-
/**
|
|
16
|
-
* Maps a user-specified distance metric to a YDB Knn function
|
|
17
|
-
* suitable for bit-quantized vectors (Phase 1 approximate candidate selection).
|
|
18
|
-
* Cosine uses similarity (DESC); other metrics use distance (ASC).
|
|
19
|
-
* For Dot, falls back to CosineDistance as a proxy since there is no
|
|
20
|
-
* direct distance equivalent for inner product.
|
|
21
|
-
*/
|
|
22
|
-
export function mapDistanceToBitKnnFn(distance) {
|
|
23
|
-
switch (distance) {
|
|
24
|
-
case "Cosine":
|
|
25
|
-
return { fn: "Knn::CosineSimilarity", order: "DESC" };
|
|
26
|
-
case "Dot":
|
|
27
|
-
return { fn: "Knn::CosineDistance", order: "ASC" };
|
|
28
|
-
case "Euclid":
|
|
29
|
-
return { fn: "Knn::EuclideanDistance", order: "ASC" };
|
|
30
|
-
case "Manhattan":
|
|
31
|
-
return { fn: "Knn::ManhattanDistance", order: "ASC" };
|
|
32
|
-
default:
|
|
33
|
-
return { fn: "Knn::CosineDistance", order: "ASC" };
|
|
34
|
-
}
|
|
35
|
-
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
function isNumericPathSegmentKey(key) {
|
|
2
|
+
return /^(0|[1-9]\d*)$/.test(key);
|
|
3
|
+
}
|
|
4
|
+
export function normalizePathSegments(value) {
|
|
5
|
+
if (Array.isArray(value)) {
|
|
6
|
+
if (value.length === 0)
|
|
7
|
+
return null;
|
|
8
|
+
const normalized = [];
|
|
9
|
+
for (const segment of value) {
|
|
10
|
+
if (typeof segment !== "string")
|
|
11
|
+
return null;
|
|
12
|
+
normalized.push(segment);
|
|
13
|
+
}
|
|
14
|
+
return normalized;
|
|
15
|
+
}
|
|
16
|
+
if (typeof value !== "object" || value === null) {
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
const entries = Object.entries(value);
|
|
20
|
+
if (entries.length === 0) {
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
const indexedSegments = [];
|
|
24
|
+
for (const [key, segment] of entries) {
|
|
25
|
+
if (!isNumericPathSegmentKey(key) || typeof segment !== "string") {
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
indexedSegments.push({ index: Number(key), value: segment });
|
|
29
|
+
}
|
|
30
|
+
indexedSegments.sort((left, right) => left.index - right.index);
|
|
31
|
+
for (let index = 0; index < indexedSegments.length; index += 1) {
|
|
32
|
+
if (indexedSegments[index]?.index !== index) {
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return indexedSegments.map((segment) => segment.value);
|
|
37
|
+
}
|
|
38
|
+
export function extractPathPrefix(payload) {
|
|
39
|
+
const pathSegments = normalizePathSegments(payload?.pathSegments);
|
|
40
|
+
if (!pathSegments) {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
return pathSegmentsToPrefix(pathSegments);
|
|
44
|
+
}
|
|
45
|
+
export function pathSegmentsToPrefix(segments) {
|
|
46
|
+
return segments.map(encodeURIComponent).join("/");
|
|
47
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function expandPathPrefixes(pathPrefix: string): string[];
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export function expandPathPrefixes(pathPrefix) {
|
|
2
|
+
if (pathPrefix === "") {
|
|
3
|
+
return [];
|
|
4
|
+
}
|
|
5
|
+
const parts = pathPrefix.split("/");
|
|
6
|
+
const prefixes = [];
|
|
7
|
+
for (let index = 1; index <= parts.length; index += 1) {
|
|
8
|
+
prefixes.push(parts.slice(0, index).join("/"));
|
|
9
|
+
}
|
|
10
|
+
return prefixes;
|
|
11
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Qdrant-compatible response envelope.
|
|
3
|
+
*
|
|
4
|
+
* Every Qdrant REST response includes `{ status, result, time, usage }`.
|
|
5
|
+
* This helper produces that shape, measuring elapsed wall-clock time
|
|
6
|
+
* from a previously captured `process.hrtime()` start mark.
|
|
7
|
+
*/
|
|
8
|
+
export declare function qdrantResponse(result: unknown, startHrTime: [number, number]): {
|
|
9
|
+
status: "ok";
|
|
10
|
+
result: unknown;
|
|
11
|
+
time: number;
|
|
12
|
+
usage: null;
|
|
13
|
+
};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Qdrant-compatible response envelope.
|
|
3
|
+
*
|
|
4
|
+
* Every Qdrant REST response includes `{ status, result, time, usage }`.
|
|
5
|
+
* This helper produces that shape, measuring elapsed wall-clock time
|
|
6
|
+
* from a previously captured `process.hrtime()` start mark.
|
|
7
|
+
*/
|
|
8
|
+
export function qdrantResponse(result, startHrTime) {
|
|
9
|
+
const diff = process.hrtime(startHrTime);
|
|
10
|
+
const time = diff[0] + diff[1] / 1e9;
|
|
11
|
+
return { status: "ok", result, time, usage: null };
|
|
12
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Request } from "express";
|
|
2
|
+
import { AnonymousIdentityError } from "./tenant.js";
|
|
3
|
+
export declare function isAnonymousIdentityError(err: unknown): err is AnonymousIdentityError;
|
|
4
|
+
export declare function getRequestApiKey(req: Request): string | undefined;
|
|
5
|
+
export declare function getRequestClientIp(req: Request): string | undefined;
|
|
6
|
+
export declare function resolveRequestUserUid(req: Request): string;
|
|
7
|
+
export declare function resolveRequestNamespaceUserUid(req: Request): string;
|
|
8
|
+
export declare function resolveRequestSigningKey(req: Request, userUid?: string): string;
|