ydb-qdrant 7.0.1 → 8.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/config/env.d.ts +0 -8
- package/dist/config/env.js +2 -29
- package/dist/package/api.d.ts +5 -2
- package/dist/package/api.js +2 -2
- package/dist/qdrant/QdrantRestTypes.d.ts +35 -0
- package/dist/repositories/collectionsRepo.d.ts +1 -2
- package/dist/repositories/collectionsRepo.js +62 -103
- package/dist/repositories/collectionsRepo.one-table.js +103 -47
- package/dist/repositories/collectionsRepo.shared.d.ts +2 -0
- package/dist/repositories/collectionsRepo.shared.js +32 -0
- package/dist/repositories/pointsRepo.d.ts +4 -8
- package/dist/repositories/pointsRepo.one-table/Delete.js +122 -67
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.d.ts +5 -2
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.js +7 -6
- package/dist/repositories/pointsRepo.one-table/Search.d.ts +4 -0
- package/dist/repositories/pointsRepo.one-table/Search.js +208 -0
- package/dist/repositories/pointsRepo.one-table/Upsert.d.ts +2 -2
- package/dist/repositories/pointsRepo.one-table/Upsert.js +51 -66
- package/dist/repositories/pointsRepo.one-table.d.ts +1 -1
- package/dist/repositories/pointsRepo.one-table.js +1 -1
- package/dist/routes/collections.js +7 -61
- package/dist/routes/points.js +11 -66
- package/dist/services/PointsService.d.ts +3 -8
- package/dist/services/PointsService.js +19 -23
- package/dist/types.d.ts +23 -33
- package/dist/types.js +18 -20
- package/dist/utils/normalization.js +13 -14
- package/dist/utils/retry.js +19 -29
- package/dist/utils/vectorBinary.js +10 -5
- package/dist/ydb/bootstrapMetaTable.d.ts +7 -0
- package/dist/ydb/bootstrapMetaTable.js +75 -0
- package/dist/ydb/client.d.ts +23 -17
- package/dist/ydb/client.js +82 -423
- package/dist/ydb/schema.js +88 -148
- package/package.json +2 -10
- package/dist/qdrant/QdrantTypes.d.ts +0 -19
- package/dist/repositories/pointsRepo.one-table/Search/Approximate.d.ts +0 -18
- package/dist/repositories/pointsRepo.one-table/Search/Approximate.js +0 -119
- package/dist/repositories/pointsRepo.one-table/Search/Exact.d.ts +0 -17
- package/dist/repositories/pointsRepo.one-table/Search/Exact.js +0 -101
- package/dist/repositories/pointsRepo.one-table/Search/index.d.ts +0 -8
- package/dist/repositories/pointsRepo.one-table/Search/index.js +0 -30
- package/dist/utils/typeGuards.d.ts +0 -1
- package/dist/utils/typeGuards.js +0 -3
- package/dist/ydb/QueryDiagnostics.d.ts +0 -6
- package/dist/ydb/QueryDiagnostics.js +0 -52
- package/dist/ydb/SessionPool.d.ts +0 -36
- package/dist/ydb/SessionPool.js +0 -248
- package/dist/ydb/bulkUpsert.d.ts +0 -6
- package/dist/ydb/bulkUpsert.js +0 -52
- /package/dist/qdrant/{QdrantTypes.js → QdrantRestTypes.js} +0 -0
|
@@ -3,67 +3,15 @@ import { putCollectionIndex, createCollection, getCollection, deleteCollection,
|
|
|
3
3
|
import { QdrantServiceError } from "../services/errors.js";
|
|
4
4
|
import { logger } from "../logging/logger.js";
|
|
5
5
|
export const collectionsRouter = Router();
|
|
6
|
-
// Placeholder defaults to satisfy Qdrant `CollectionInfo` shape. These values are
|
|
7
|
-
// not used for execution in ydb-qdrant, only for client compatibility.
|
|
8
|
-
const DEFAULT_HNSW_CONFIG = {
|
|
9
|
-
m: 16,
|
|
10
|
-
ef_construct: 100,
|
|
11
|
-
full_scan_threshold: 10000,
|
|
12
|
-
max_indexing_threads: 0,
|
|
13
|
-
on_disk: false,
|
|
14
|
-
};
|
|
15
|
-
const DEFAULT_OPTIMIZERS_CONFIG = {
|
|
16
|
-
deleted_threshold: 0.2,
|
|
17
|
-
vacuum_min_vector_number: 1000,
|
|
18
|
-
default_segment_number: 0,
|
|
19
|
-
indexing_threshold: 10000,
|
|
20
|
-
flush_interval_sec: 5,
|
|
21
|
-
};
|
|
22
|
-
function mapVectorDatatype(dataType) {
|
|
23
|
-
// Our service exposes `float`; Qdrant uses `float32`/`float16`/`uint8`.
|
|
24
|
-
if (dataType === "float16")
|
|
25
|
-
return "float16";
|
|
26
|
-
if (dataType === "uint8")
|
|
27
|
-
return "uint8";
|
|
28
|
-
return "float32";
|
|
29
|
-
}
|
|
30
|
-
function toQdrantCollectionInfo(result) {
|
|
31
|
-
const vectors = result.vectors;
|
|
32
|
-
const datatype = mapVectorDatatype(vectors?.data_type);
|
|
33
|
-
const config = {
|
|
34
|
-
params: {
|
|
35
|
-
vectors: {
|
|
36
|
-
size: vectors.size,
|
|
37
|
-
distance: vectors.distance,
|
|
38
|
-
datatype,
|
|
39
|
-
on_disk: false,
|
|
40
|
-
},
|
|
41
|
-
shard_number: 1,
|
|
42
|
-
replication_factor: 1,
|
|
43
|
-
write_consistency_factor: 1,
|
|
44
|
-
on_disk_payload: false,
|
|
45
|
-
},
|
|
46
|
-
hnsw_config: DEFAULT_HNSW_CONFIG,
|
|
47
|
-
optimizer_config: DEFAULT_OPTIMIZERS_CONFIG,
|
|
48
|
-
};
|
|
49
|
-
return {
|
|
50
|
-
status: "green",
|
|
51
|
-
optimizer_status: "ok",
|
|
52
|
-
segments_count: 1,
|
|
53
|
-
config,
|
|
54
|
-
payload_schema: {},
|
|
55
|
-
};
|
|
56
|
-
}
|
|
57
6
|
collectionsRouter.put("/:collection/index", async (req, res) => {
|
|
58
7
|
try {
|
|
59
|
-
await putCollectionIndex({
|
|
8
|
+
const result = await putCollectionIndex({
|
|
60
9
|
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
61
10
|
collection: String(req.params.collection),
|
|
62
11
|
apiKey: req.header("api-key") ?? undefined,
|
|
63
12
|
userAgent: req.header("User-Agent") ?? undefined,
|
|
64
13
|
});
|
|
65
|
-
|
|
66
|
-
res.json({ status: "ok", result: true });
|
|
14
|
+
res.json({ status: "ok", result });
|
|
67
15
|
}
|
|
68
16
|
catch (err) {
|
|
69
17
|
if (err instanceof QdrantServiceError) {
|
|
@@ -76,14 +24,13 @@ collectionsRouter.put("/:collection/index", async (req, res) => {
|
|
|
76
24
|
});
|
|
77
25
|
collectionsRouter.put("/:collection", async (req, res) => {
|
|
78
26
|
try {
|
|
79
|
-
await createCollection({
|
|
27
|
+
const result = await createCollection({
|
|
80
28
|
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
81
29
|
collection: String(req.params.collection),
|
|
82
30
|
apiKey: req.header("api-key") ?? undefined,
|
|
83
31
|
userAgent: req.header("User-Agent") ?? undefined,
|
|
84
32
|
}, req.body);
|
|
85
|
-
|
|
86
|
-
res.json({ status: "ok", result: true });
|
|
33
|
+
res.json({ status: "ok", result });
|
|
87
34
|
}
|
|
88
35
|
catch (err) {
|
|
89
36
|
if (err instanceof QdrantServiceError) {
|
|
@@ -102,7 +49,7 @@ collectionsRouter.get("/:collection", async (req, res) => {
|
|
|
102
49
|
apiKey: req.header("api-key") ?? undefined,
|
|
103
50
|
userAgent: req.header("User-Agent") ?? undefined,
|
|
104
51
|
});
|
|
105
|
-
res.json({ status: "ok", result
|
|
52
|
+
res.json({ status: "ok", result });
|
|
106
53
|
}
|
|
107
54
|
catch (err) {
|
|
108
55
|
if (err instanceof QdrantServiceError) {
|
|
@@ -115,14 +62,13 @@ collectionsRouter.get("/:collection", async (req, res) => {
|
|
|
115
62
|
});
|
|
116
63
|
collectionsRouter.delete("/:collection", async (req, res) => {
|
|
117
64
|
try {
|
|
118
|
-
await deleteCollection({
|
|
65
|
+
const result = await deleteCollection({
|
|
119
66
|
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
120
67
|
collection: String(req.params.collection),
|
|
121
68
|
apiKey: req.header("api-key") ?? undefined,
|
|
122
69
|
userAgent: req.header("User-Agent") ?? undefined,
|
|
123
70
|
});
|
|
124
|
-
|
|
125
|
-
res.json({ status: "ok", result: true });
|
|
71
|
+
res.json({ status: "ok", result });
|
|
126
72
|
}
|
|
127
73
|
catch (err) {
|
|
128
74
|
if (err instanceof QdrantServiceError) {
|
package/dist/routes/points.js
CHANGED
|
@@ -2,18 +2,17 @@ import { Router } from "express";
|
|
|
2
2
|
import { upsertPoints, searchPoints, queryPoints, deletePoints, } from "../services/PointsService.js";
|
|
3
3
|
import { QdrantServiceError } from "../services/errors.js";
|
|
4
4
|
import { logger } from "../logging/logger.js";
|
|
5
|
-
import {
|
|
6
|
-
import { getAbortErrorCause, isCompilationTimeoutError, isTimeoutAbortError, } from "../ydb/client.js";
|
|
5
|
+
import { isCompilationTimeoutError } from "../ydb/client.js";
|
|
7
6
|
import { scheduleExit } from "../utils/exit.js";
|
|
8
7
|
export const pointsRouter = Router();
|
|
9
|
-
function toQdrantScoredPoint(
|
|
10
|
-
//
|
|
11
|
-
//
|
|
8
|
+
function toQdrantScoredPoint(p) {
|
|
9
|
+
// We don't currently track per-point versions or return vectors/shard keys,
|
|
10
|
+
// but many Qdrant clients expect these fields to exist in the response.
|
|
12
11
|
return {
|
|
13
|
-
id:
|
|
12
|
+
id: p.id,
|
|
14
13
|
version: 0,
|
|
15
|
-
score:
|
|
16
|
-
payload:
|
|
14
|
+
score: p.score,
|
|
15
|
+
payload: p.payload ?? null,
|
|
17
16
|
vector: null,
|
|
18
17
|
shard_key: null,
|
|
19
18
|
order_value: null,
|
|
@@ -41,17 +40,6 @@ pointsRouter.put("/:collection/points", async (req, res) => {
|
|
|
41
40
|
scheduleExit(1);
|
|
42
41
|
return;
|
|
43
42
|
}
|
|
44
|
-
if (isTimeoutAbortError(err)) {
|
|
45
|
-
logger.error({
|
|
46
|
-
err,
|
|
47
|
-
errCause: getAbortErrorCause(err),
|
|
48
|
-
timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
|
|
49
|
-
}, "YDB upsert operation timed out");
|
|
50
|
-
return res.status(500).json({
|
|
51
|
-
status: "error",
|
|
52
|
-
error: `upsert operation timed out after ${UPSERT_OPERATION_TIMEOUT_MS}ms`,
|
|
53
|
-
});
|
|
54
|
-
}
|
|
55
43
|
logger.error({ err }, "upsert points (PUT) failed");
|
|
56
44
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
57
45
|
}
|
|
@@ -77,35 +65,19 @@ pointsRouter.post("/:collection/points/upsert", async (req, res) => {
|
|
|
77
65
|
scheduleExit(1);
|
|
78
66
|
return;
|
|
79
67
|
}
|
|
80
|
-
if (isTimeoutAbortError(err)) {
|
|
81
|
-
logger.error({
|
|
82
|
-
err,
|
|
83
|
-
errCause: getAbortErrorCause(err),
|
|
84
|
-
timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
|
|
85
|
-
}, "YDB upsert operation timed out");
|
|
86
|
-
return res.status(500).json({
|
|
87
|
-
status: "error",
|
|
88
|
-
error: `upsert operation timed out after ${UPSERT_OPERATION_TIMEOUT_MS}ms`,
|
|
89
|
-
});
|
|
90
|
-
}
|
|
91
68
|
logger.error({ err }, "upsert points failed");
|
|
92
69
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
93
70
|
}
|
|
94
71
|
});
|
|
95
72
|
pointsRouter.post("/:collection/points/search", async (req, res) => {
|
|
96
73
|
try {
|
|
97
|
-
const
|
|
74
|
+
const { points } = await searchPoints({
|
|
98
75
|
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
99
76
|
collection: String(req.params.collection),
|
|
100
77
|
apiKey: req.header("api-key") ?? undefined,
|
|
101
78
|
userAgent: req.header("User-Agent") ?? undefined,
|
|
102
79
|
}, req.body);
|
|
103
|
-
|
|
104
|
-
// Keep service return shape internal (`{ points: [...] }`).
|
|
105
|
-
res.json({
|
|
106
|
-
status: "ok",
|
|
107
|
-
result: result.points.map(toQdrantScoredPoint),
|
|
108
|
-
});
|
|
80
|
+
res.json({ status: "ok", result: points.map(toQdrantScoredPoint) });
|
|
109
81
|
}
|
|
110
82
|
catch (err) {
|
|
111
83
|
if (err instanceof QdrantServiceError) {
|
|
@@ -118,17 +90,6 @@ pointsRouter.post("/:collection/points/search", async (req, res) => {
|
|
|
118
90
|
scheduleExit(1);
|
|
119
91
|
return;
|
|
120
92
|
}
|
|
121
|
-
if (isTimeoutAbortError(err)) {
|
|
122
|
-
logger.error({
|
|
123
|
-
err,
|
|
124
|
-
errCause: getAbortErrorCause(err),
|
|
125
|
-
timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
|
|
126
|
-
}, "YDB search operation timed out");
|
|
127
|
-
return res.status(500).json({
|
|
128
|
-
status: "error",
|
|
129
|
-
error: `search operation timed out after ${SEARCH_OPERATION_TIMEOUT_MS}ms`,
|
|
130
|
-
});
|
|
131
|
-
}
|
|
132
93
|
logger.error({ err }, "search points failed");
|
|
133
94
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
134
95
|
}
|
|
@@ -136,18 +97,13 @@ pointsRouter.post("/:collection/points/search", async (req, res) => {
|
|
|
136
97
|
// Compatibility: some clients call POST /collections/:collection/points/query
|
|
137
98
|
pointsRouter.post("/:collection/points/query", async (req, res) => {
|
|
138
99
|
try {
|
|
139
|
-
const
|
|
100
|
+
const { points } = await queryPoints({
|
|
140
101
|
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
141
102
|
collection: String(req.params.collection),
|
|
142
103
|
apiKey: req.header("api-key") ?? undefined,
|
|
143
104
|
userAgent: req.header("User-Agent") ?? undefined,
|
|
144
105
|
}, req.body);
|
|
145
|
-
|
|
146
|
-
// (Unlike /points/search, where result is a list.)
|
|
147
|
-
const qdrantResult = {
|
|
148
|
-
points: result.points.map(toQdrantScoredPoint),
|
|
149
|
-
};
|
|
150
|
-
res.json({ status: "ok", result: qdrantResult });
|
|
106
|
+
res.json({ status: "ok", result: points.map(toQdrantScoredPoint) });
|
|
151
107
|
}
|
|
152
108
|
catch (err) {
|
|
153
109
|
if (err instanceof QdrantServiceError) {
|
|
@@ -160,17 +116,6 @@ pointsRouter.post("/:collection/points/query", async (req, res) => {
|
|
|
160
116
|
scheduleExit(1);
|
|
161
117
|
return;
|
|
162
118
|
}
|
|
163
|
-
if (isTimeoutAbortError(err)) {
|
|
164
|
-
logger.error({
|
|
165
|
-
err,
|
|
166
|
-
errCause: getAbortErrorCause(err),
|
|
167
|
-
timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
|
|
168
|
-
}, "YDB search operation timed out");
|
|
169
|
-
return res.status(500).json({
|
|
170
|
-
status: "error",
|
|
171
|
-
error: `search operation timed out after ${SEARCH_OPERATION_TIMEOUT_MS}ms`,
|
|
172
|
-
});
|
|
173
|
-
}
|
|
174
119
|
logger.error({ err }, "search points (query) failed");
|
|
175
120
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
176
121
|
}
|
|
@@ -1,19 +1,14 @@
|
|
|
1
1
|
import { type CollectionContextInput } from "./CollectionService.js";
|
|
2
|
-
import type {
|
|
2
|
+
import type { YdbQdrantScoredPoint } from "../qdrant/QdrantRestTypes.js";
|
|
3
3
|
type PointsContextInput = CollectionContextInput;
|
|
4
|
-
type InternalScoredPoint = {
|
|
5
|
-
id: string;
|
|
6
|
-
score: number;
|
|
7
|
-
payload?: QdrantPayload;
|
|
8
|
-
};
|
|
9
4
|
export declare function upsertPoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
10
5
|
upserted: number;
|
|
11
6
|
}>;
|
|
12
7
|
export declare function searchPoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
13
|
-
points:
|
|
8
|
+
points: YdbQdrantScoredPoint[];
|
|
14
9
|
}>;
|
|
15
10
|
export declare function queryPoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
16
|
-
points:
|
|
11
|
+
points: YdbQdrantScoredPoint[];
|
|
17
12
|
}>;
|
|
18
13
|
export declare function deletePoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
19
14
|
deleted: number;
|
|
@@ -7,31 +7,29 @@ import { QdrantServiceError, isVectorDimensionMismatchError, } from "./errors.js
|
|
|
7
7
|
import { normalizeCollectionContextShared } from "./CollectionService.shared.js";
|
|
8
8
|
import { resolvePointsTableAndUidOneTable } from "./CollectionService.one-table.js";
|
|
9
9
|
import { normalizeSearchBodyForSearch, normalizeSearchBodyForQuery, } from "../utils/normalization.js";
|
|
10
|
-
import { isRecord } from "../utils/typeGuards.js";
|
|
11
10
|
function parsePathSegmentsFilterToPaths(filter) {
|
|
12
11
|
const extractMust = (must) => {
|
|
13
12
|
if (!Array.isArray(must) || must.length === 0)
|
|
14
13
|
return null;
|
|
15
14
|
const pairs = [];
|
|
16
15
|
for (const cond of must) {
|
|
17
|
-
if (
|
|
16
|
+
if (typeof cond !== "object" || cond === null)
|
|
18
17
|
return null;
|
|
19
|
-
const
|
|
20
|
-
if (typeof key !== "string")
|
|
18
|
+
const c = cond;
|
|
19
|
+
if (typeof c.key !== "string")
|
|
21
20
|
return null;
|
|
22
|
-
const m = /^pathSegments\.(\d+)$/.exec(key);
|
|
21
|
+
const m = /^pathSegments\.(\d+)$/.exec(c.key);
|
|
23
22
|
if (!m)
|
|
24
23
|
return null;
|
|
25
24
|
const idx = Number(m[1]);
|
|
26
25
|
if (!Number.isInteger(idx) || idx < 0)
|
|
27
26
|
return null;
|
|
28
|
-
|
|
29
|
-
if (!isRecord(match))
|
|
27
|
+
if (typeof c.match !== "object" || c.match === null)
|
|
30
28
|
return null;
|
|
31
|
-
const
|
|
32
|
-
if (typeof value !== "string")
|
|
29
|
+
const match = c.match;
|
|
30
|
+
if (typeof match.value !== "string")
|
|
33
31
|
return null;
|
|
34
|
-
pairs.push({ idx, value });
|
|
32
|
+
pairs.push({ idx, value: match.value });
|
|
35
33
|
}
|
|
36
34
|
pairs.sort((a, b) => a.idx - b.idx);
|
|
37
35
|
// Require contiguous indexes starting from 0 to avoid ambiguous matches.
|
|
@@ -41,22 +39,22 @@ function parsePathSegmentsFilterToPaths(filter) {
|
|
|
41
39
|
}
|
|
42
40
|
return pairs.map((p) => p.value);
|
|
43
41
|
};
|
|
44
|
-
if (
|
|
42
|
+
if (typeof filter !== "object" || filter === null)
|
|
45
43
|
return null;
|
|
46
|
-
const
|
|
47
|
-
if (must !== undefined) {
|
|
48
|
-
const path = extractMust(must);
|
|
44
|
+
const f = filter;
|
|
45
|
+
if (f.must !== undefined) {
|
|
46
|
+
const path = extractMust(f.must);
|
|
49
47
|
return path ? [path] : null;
|
|
50
48
|
}
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
if (!Array.isArray(should) || should.length === 0)
|
|
49
|
+
if (f.should !== undefined) {
|
|
50
|
+
if (!Array.isArray(f.should) || f.should.length === 0)
|
|
54
51
|
return null;
|
|
55
52
|
const paths = [];
|
|
56
|
-
for (const g of should) {
|
|
57
|
-
if (
|
|
53
|
+
for (const g of f.should) {
|
|
54
|
+
if (typeof g !== "object" || g === null)
|
|
58
55
|
return null;
|
|
59
|
-
const
|
|
56
|
+
const group = g;
|
|
57
|
+
const path = extractMust(group.must);
|
|
60
58
|
if (!path)
|
|
61
59
|
return null;
|
|
62
60
|
paths.push(path);
|
|
@@ -85,9 +83,7 @@ export async function upsertPoints(ctx, body) {
|
|
|
85
83
|
const { tableName, uid } = await resolvePointsTableAndUidOneTable(normalized);
|
|
86
84
|
let upserted;
|
|
87
85
|
try {
|
|
88
|
-
|
|
89
|
-
const points = parsed.data.points;
|
|
90
|
-
upserted = await repoUpsertPoints(tableName, points, meta.dimension, uid);
|
|
86
|
+
upserted = await repoUpsertPoints(tableName, parsed.data.points, meta.dimension, uid);
|
|
91
87
|
}
|
|
92
88
|
catch (err) {
|
|
93
89
|
if (isVectorDimensionMismatchError(err)) {
|
package/dist/types.d.ts
CHANGED
|
@@ -1,18 +1,9 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import type { QdrantDistance,
|
|
3
|
-
export declare const DistanceKindSchema: z.ZodEnum<{
|
|
4
|
-
Cosine: "Cosine";
|
|
5
|
-
Euclid: "Euclid";
|
|
6
|
-
Dot: "Dot";
|
|
7
|
-
Manhattan: "Manhattan";
|
|
8
|
-
}>;
|
|
2
|
+
import type { QdrantDistance, QdrantPayload, QdrantWithPayloadInterface, YdbQdrantPointId, YdbQdrantUpsertPoint } from "./qdrant/QdrantRestTypes.js";
|
|
9
3
|
export type DistanceKind = QdrantDistance;
|
|
10
|
-
export
|
|
11
|
-
export type
|
|
12
|
-
export
|
|
13
|
-
export type PointId = QdrantPointId;
|
|
14
|
-
export declare const DenseVectorSchema: z.ZodArray<z.ZodNumber>;
|
|
15
|
-
export type DenseVector = QdrantDenseVector;
|
|
4
|
+
export type VectorType = "float";
|
|
5
|
+
export type Payload = QdrantPayload;
|
|
6
|
+
export type WithPayload = QdrantWithPayloadInterface;
|
|
16
7
|
/**
|
|
17
8
|
* Collection metadata from qdr__collections table.
|
|
18
9
|
*
|
|
@@ -29,34 +20,37 @@ export interface CollectionMeta {
|
|
|
29
20
|
export declare const CreateCollectionReq: z.ZodObject<{
|
|
30
21
|
vectors: z.ZodObject<{
|
|
31
22
|
size: z.ZodNumber;
|
|
32
|
-
distance: z.
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
Manhattan: "Manhattan";
|
|
37
|
-
}>;
|
|
38
|
-
data_type: z.ZodOptional<z.ZodLiteral<"float">>;
|
|
23
|
+
distance: z.ZodType<DistanceKind>;
|
|
24
|
+
data_type: z.ZodOptional<z.ZodEnum<{
|
|
25
|
+
float: "float";
|
|
26
|
+
}>>;
|
|
39
27
|
}, z.core.$strip>;
|
|
40
28
|
}, z.core.$strip>;
|
|
41
|
-
export declare const UpsertPointSchema: z.ZodObject<{
|
|
42
|
-
id: z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>;
|
|
43
|
-
vector: z.ZodArray<z.ZodNumber>;
|
|
44
|
-
payload: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
45
|
-
}, z.core.$strip>;
|
|
46
29
|
export declare const UpsertPointsReq: z.ZodObject<{
|
|
47
30
|
points: z.ZodArray<z.ZodObject<{
|
|
48
|
-
id: z.
|
|
31
|
+
id: z.ZodType<YdbQdrantPointId>;
|
|
49
32
|
vector: z.ZodArray<z.ZodNumber>;
|
|
50
|
-
payload: z.
|
|
33
|
+
payload: z.ZodType<Payload | undefined>;
|
|
51
34
|
}, z.core.$strip>>;
|
|
52
35
|
}, z.core.$strip>;
|
|
36
|
+
export type UpsertPoint = YdbQdrantUpsertPoint;
|
|
37
|
+
export type UpsertPointsBody = {
|
|
38
|
+
points: UpsertPoint[];
|
|
39
|
+
};
|
|
53
40
|
export declare const SearchReq: z.ZodObject<{
|
|
54
41
|
vector: z.ZodArray<z.ZodNumber>;
|
|
55
42
|
top: z.ZodNumber;
|
|
56
43
|
with_payload: z.ZodOptional<z.ZodBoolean>;
|
|
57
44
|
}, z.core.$strip>;
|
|
45
|
+
export type SearchPointsBody = {
|
|
46
|
+
vector: number[];
|
|
47
|
+
top?: number;
|
|
48
|
+
limit?: number;
|
|
49
|
+
with_payload?: WithPayload;
|
|
50
|
+
score_threshold?: number | null;
|
|
51
|
+
};
|
|
58
52
|
export declare const DeletePointsByIdsReq: z.ZodObject<{
|
|
59
|
-
points: z.ZodArray<z.
|
|
53
|
+
points: z.ZodArray<z.ZodType<YdbQdrantPointId, unknown, z.core.$ZodTypeInternals<YdbQdrantPointId, unknown>>>;
|
|
60
54
|
}, z.core.$strip>;
|
|
61
55
|
export declare const DeletePointsByFilterReq: z.ZodObject<{
|
|
62
56
|
filter: z.ZodUnion<readonly [z.ZodObject<{
|
|
@@ -78,7 +72,7 @@ export declare const DeletePointsByFilterReq: z.ZodObject<{
|
|
|
78
72
|
}, z.core.$strip>]>;
|
|
79
73
|
}, z.core.$strip>;
|
|
80
74
|
export declare const DeletePointsReq: z.ZodUnion<readonly [z.ZodObject<{
|
|
81
|
-
points: z.ZodArray<z.
|
|
75
|
+
points: z.ZodArray<z.ZodType<YdbQdrantPointId, unknown, z.core.$ZodTypeInternals<YdbQdrantPointId, unknown>>>;
|
|
82
76
|
}, z.core.$strip>, z.ZodObject<{
|
|
83
77
|
filter: z.ZodUnion<readonly [z.ZodObject<{
|
|
84
78
|
must: z.ZodArray<z.ZodObject<{
|
|
@@ -98,7 +92,3 @@ export declare const DeletePointsReq: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
98
92
|
}, z.core.$strip>>;
|
|
99
93
|
}, z.core.$strip>]>;
|
|
100
94
|
}, z.core.$strip>]>;
|
|
101
|
-
export type CreateCollectionBody = z.infer<typeof CreateCollectionReq>;
|
|
102
|
-
export type UpsertPointsBody = z.infer<typeof UpsertPointsReq>;
|
|
103
|
-
export type SearchBody = z.infer<typeof SearchReq>;
|
|
104
|
-
export type DeletePointsBody = z.infer<typeof DeletePointsReq>;
|
package/dist/types.js
CHANGED
|
@@ -1,36 +1,34 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
const DISTANCE_KIND_VALUES = [
|
|
3
|
-
"Cosine",
|
|
4
|
-
"Euclid",
|
|
5
|
-
"Dot",
|
|
6
|
-
"Manhattan",
|
|
7
|
-
];
|
|
8
|
-
export const DistanceKindSchema = z.enum(DISTANCE_KIND_VALUES);
|
|
9
|
-
export const VectorTypeSchema = z.literal("float");
|
|
10
|
-
export const PointIdSchema = z.union([z.string(), z.number()]);
|
|
11
|
-
export const DenseVectorSchema = z.array(z.number());
|
|
12
2
|
export const CreateCollectionReq = z.object({
|
|
13
3
|
vectors: z.object({
|
|
14
4
|
size: z.number().int().positive(),
|
|
15
|
-
distance:
|
|
16
|
-
|
|
5
|
+
distance: z.enum([
|
|
6
|
+
"Cosine",
|
|
7
|
+
"Euclid",
|
|
8
|
+
"Dot",
|
|
9
|
+
"Manhattan",
|
|
10
|
+
]),
|
|
11
|
+
data_type: z.enum(["float"]).optional(),
|
|
17
12
|
}),
|
|
18
13
|
});
|
|
19
|
-
export const UpsertPointSchema = z.object({
|
|
20
|
-
id: PointIdSchema,
|
|
21
|
-
vector: DenseVectorSchema,
|
|
22
|
-
payload: z.record(z.string(), z.unknown()).optional(),
|
|
23
|
-
});
|
|
24
14
|
export const UpsertPointsReq = z.object({
|
|
25
|
-
points: z
|
|
15
|
+
points: z
|
|
16
|
+
.array(z.object({
|
|
17
|
+
id: z.union([z.string(), z.number()]),
|
|
18
|
+
vector: z.array(z.number()),
|
|
19
|
+
payload: z.record(z.string(), z.unknown()).optional(),
|
|
20
|
+
}))
|
|
21
|
+
.min(1),
|
|
26
22
|
});
|
|
27
23
|
export const SearchReq = z.object({
|
|
28
|
-
vector:
|
|
24
|
+
vector: z.array(z.number()).min(1),
|
|
29
25
|
top: z.number().int().positive().max(1000),
|
|
30
26
|
with_payload: z.boolean().optional(),
|
|
31
27
|
});
|
|
32
28
|
export const DeletePointsByIdsReq = z.object({
|
|
33
|
-
points: z
|
|
29
|
+
points: z
|
|
30
|
+
.array(z.union([z.string(), z.number()]))
|
|
31
|
+
.min(1),
|
|
34
32
|
});
|
|
35
33
|
const DeletePointsFilterCondition = z.object({
|
|
36
34
|
key: z.string(),
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import { isRecord } from "./typeGuards.js";
|
|
2
1
|
export function isNumberArray(value) {
|
|
3
2
|
return Array.isArray(value) && value.every((x) => typeof x === "number");
|
|
4
3
|
}
|
|
5
4
|
export function extractVectorLoose(body, depth = 0) {
|
|
6
|
-
if (!
|
|
5
|
+
if (!body || typeof body !== "object" || depth > 3) {
|
|
7
6
|
return undefined;
|
|
8
7
|
}
|
|
9
8
|
const obj = body;
|
|
@@ -11,17 +10,17 @@ export function extractVectorLoose(body, depth = 0) {
|
|
|
11
10
|
return obj.vector;
|
|
12
11
|
if (isNumberArray(obj.embedding))
|
|
13
12
|
return obj.embedding;
|
|
14
|
-
const query =
|
|
13
|
+
const query = obj.query;
|
|
15
14
|
if (query) {
|
|
16
|
-
const queryVector = query
|
|
15
|
+
const queryVector = query["vector"];
|
|
17
16
|
if (isNumberArray(queryVector))
|
|
18
17
|
return queryVector;
|
|
19
|
-
const nearest =
|
|
18
|
+
const nearest = query["nearest"];
|
|
20
19
|
if (nearest && isNumberArray(nearest.vector)) {
|
|
21
20
|
return nearest.vector;
|
|
22
21
|
}
|
|
23
22
|
}
|
|
24
|
-
const nearest =
|
|
23
|
+
const nearest = obj.nearest;
|
|
25
24
|
if (nearest && isNumberArray(nearest.vector)) {
|
|
26
25
|
return nearest.vector;
|
|
27
26
|
}
|
|
@@ -43,7 +42,7 @@ export function extractVectorLoose(body, depth = 0) {
|
|
|
43
42
|
return undefined;
|
|
44
43
|
}
|
|
45
44
|
export function normalizeSearchBodyForSearch(body) {
|
|
46
|
-
if (!
|
|
45
|
+
if (!body || typeof body !== "object") {
|
|
47
46
|
return {
|
|
48
47
|
vector: undefined,
|
|
49
48
|
top: undefined,
|
|
@@ -52,12 +51,12 @@ export function normalizeSearchBodyForSearch(body) {
|
|
|
52
51
|
};
|
|
53
52
|
}
|
|
54
53
|
const b = body;
|
|
55
|
-
const rawVector = b
|
|
54
|
+
const rawVector = b["vector"];
|
|
56
55
|
const vector = isNumberArray(rawVector) ? rawVector : undefined;
|
|
57
56
|
return normalizeSearchCommon(b, vector);
|
|
58
57
|
}
|
|
59
58
|
export function normalizeSearchBodyForQuery(body) {
|
|
60
|
-
if (!
|
|
59
|
+
if (!body || typeof body !== "object") {
|
|
61
60
|
return {
|
|
62
61
|
vector: undefined,
|
|
63
62
|
top: undefined,
|
|
@@ -70,14 +69,14 @@ export function normalizeSearchBodyForQuery(body) {
|
|
|
70
69
|
return normalizeSearchCommon(b, vector);
|
|
71
70
|
}
|
|
72
71
|
function normalizeSearchCommon(b, vector) {
|
|
73
|
-
const rawTop = b
|
|
74
|
-
const rawLimit = b
|
|
72
|
+
const rawTop = b["top"];
|
|
73
|
+
const rawLimit = b["limit"];
|
|
75
74
|
const topFromTop = typeof rawTop === "number" ? rawTop : undefined;
|
|
76
75
|
const topFromLimit = typeof rawLimit === "number" ? rawLimit : undefined;
|
|
77
76
|
const top = topFromTop ?? topFromLimit;
|
|
78
|
-
const filter = b
|
|
77
|
+
const filter = b["filter"];
|
|
79
78
|
let withPayload;
|
|
80
|
-
const rawWithPayload = b
|
|
79
|
+
const rawWithPayload = b["with_payload"];
|
|
81
80
|
if (typeof rawWithPayload === "boolean") {
|
|
82
81
|
withPayload = rawWithPayload;
|
|
83
82
|
}
|
|
@@ -85,7 +84,7 @@ function normalizeSearchCommon(b, vector) {
|
|
|
85
84
|
typeof rawWithPayload === "object") {
|
|
86
85
|
withPayload = true;
|
|
87
86
|
}
|
|
88
|
-
const thresholdRaw = b
|
|
87
|
+
const thresholdRaw = b["score_threshold"];
|
|
89
88
|
const thresholdValue = typeof thresholdRaw === "number" ? thresholdRaw : Number(thresholdRaw);
|
|
90
89
|
const scoreThreshold = Number.isFinite(thresholdValue)
|
|
91
90
|
? thresholdValue
|
package/dist/utils/retry.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { logger } from "../logging/logger.js";
|
|
2
|
-
import { retry as ydbRetry } from "@ydbjs/retry";
|
|
3
2
|
const DEFAULT_MAX_RETRIES = 6;
|
|
4
3
|
const DEFAULT_BASE_DELAY_MS = 250;
|
|
5
4
|
export function isTransientYdbError(error) {
|
|
@@ -23,35 +22,26 @@ export async function withRetry(fn, options = {}) {
|
|
|
23
22
|
const baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;
|
|
24
23
|
const isTransient = options.isTransient ?? isTransientYdbError;
|
|
25
24
|
const context = options.context ?? {};
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
const attemptIndex = Math.max(0, ctx.attempt - 1);
|
|
37
|
-
const delayMs = Math.floor(baseDelayMs * Math.pow(2, attemptIndex) + Math.random() * 100);
|
|
38
|
-
delayByAttempt.set(ctx.attempt, delayMs);
|
|
39
|
-
return delayMs;
|
|
40
|
-
},
|
|
41
|
-
onRetry: (ctx) => {
|
|
42
|
-
const attemptIndex = Math.max(0, ctx.attempt - 1);
|
|
25
|
+
let attempt = 0;
|
|
26
|
+
while (true) {
|
|
27
|
+
try {
|
|
28
|
+
return await fn();
|
|
29
|
+
}
|
|
30
|
+
catch (e) {
|
|
31
|
+
if (!isTransient(e) || attempt >= maxRetries) {
|
|
32
|
+
throw e;
|
|
33
|
+
}
|
|
34
|
+
const backoffMs = Math.floor(baseDelayMs * Math.pow(2, attempt) + Math.random() * 100);
|
|
43
35
|
logger.warn({
|
|
44
36
|
...context,
|
|
45
|
-
attempt
|
|
46
|
-
backoffMs
|
|
47
|
-
err:
|
|
48
|
-
?
|
|
49
|
-
: new Error(typeof
|
|
50
|
-
? ctx.error
|
|
51
|
-
: JSON.stringify(ctx.error)),
|
|
37
|
+
attempt,
|
|
38
|
+
backoffMs,
|
|
39
|
+
err: e instanceof Error
|
|
40
|
+
? e
|
|
41
|
+
: new Error(typeof e === "string" ? e : JSON.stringify(e)),
|
|
52
42
|
}, "operation aborted due to transient error; retrying");
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
}
|
|
43
|
+
await new Promise((r) => setTimeout(r, backoffMs));
|
|
44
|
+
attempt += 1;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
57
47
|
}
|