ydb-qdrant 8.1.0 → 9.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -18
- package/dist/SmokeTest.js +2 -2
- package/dist/compute/ComputePool.d.ts +5 -0
- package/dist/compute/ComputePool.js +64 -0
- package/dist/compute/ComputeWorker.d.ts +36 -0
- package/dist/compute/ComputeWorker.js +84 -0
- package/dist/config/env.d.ts +24 -7
- package/dist/config/env.js +65 -35
- package/dist/index.d.ts +2 -0
- package/dist/index.js +92 -2
- package/dist/logging/DeployLogFormatter.d.ts +2 -0
- package/dist/logging/DeployLogFormatter.js +131 -0
- package/dist/logging/logger.js +13 -1
- package/dist/logging/requestContext.d.ts +17 -0
- package/dist/logging/requestContext.js +43 -0
- package/dist/middleware/requestLogger.js +134 -6
- package/dist/middleware/upsertBodyPhase.d.ts +6 -0
- package/dist/middleware/upsertBodyPhase.js +184 -0
- package/dist/middleware/upsertRequestTimeout.d.ts +16 -0
- package/dist/middleware/upsertRequestTimeout.js +158 -0
- package/dist/package/api.d.ts +20 -12
- package/dist/package/api.js +57 -28
- package/dist/qdrant/QdrantRestTypes.d.ts +4 -0
- package/dist/qdrant/Requests.d.ts +97 -0
- package/dist/qdrant/Requests.js +72 -0
- package/dist/repositories/collectionsRepo.d.ts +18 -2
- package/dist/repositories/collectionsRepo.js +103 -7
- package/dist/repositories/collectionsRepo.one-table.d.ts +4 -3
- package/dist/repositories/collectionsRepo.one-table.js +99 -36
- package/dist/repositories/collectionsRepo.shared.d.ts +2 -2
- package/dist/repositories/collectionsRepo.shared.js +9 -4
- package/dist/repositories/pointsRepo.d.ts +6 -4
- package/dist/repositories/pointsRepo.js +8 -7
- package/dist/repositories/pointsRepo.one-table/Delete.d.ts +2 -2
- package/dist/repositories/pointsRepo.one-table/Delete.js +157 -60
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.d.ts +7 -5
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.js +44 -13
- package/dist/repositories/pointsRepo.one-table/Retrieve.d.ts +6 -0
- package/dist/repositories/pointsRepo.one-table/Retrieve.js +69 -0
- package/dist/repositories/pointsRepo.one-table/Search.d.ts +2 -3
- package/dist/repositories/pointsRepo.one-table/Search.js +102 -124
- package/dist/repositories/pointsRepo.one-table/Upsert.d.ts +2 -2
- package/dist/repositories/pointsRepo.one-table/Upsert.js +244 -48
- package/dist/repositories/pointsRepo.one-table.d.ts +1 -0
- package/dist/repositories/pointsRepo.one-table.js +1 -0
- package/dist/routes/collections.js +45 -36
- package/dist/routes/points.js +145 -56
- package/dist/server.js +42 -6
- package/dist/services/CollectionService.d.ts +7 -5
- package/dist/services/CollectionService.js +12 -9
- package/dist/services/CollectionService.one-table.js +1 -2
- package/dist/services/CollectionService.shared.d.ts +6 -5
- package/dist/services/CollectionService.shared.js +28 -12
- package/dist/services/PointsService.d.ts +8 -0
- package/dist/services/PointsService.js +132 -15
- package/dist/types.d.ts +4 -94
- package/dist/types.js +1 -54
- package/dist/utils/EnvParsers.d.ts +5 -0
- package/dist/utils/EnvParsers.js +30 -0
- package/dist/utils/PayloadSign.d.ts +4 -0
- package/dist/utils/PayloadSign.js +18 -0
- package/dist/utils/distance.d.ts +1 -12
- package/dist/utils/distance.js +0 -21
- package/dist/utils/pathPrefix.d.ts +3 -0
- package/dist/utils/pathPrefix.js +47 -0
- package/dist/utils/prefixExpansion.d.ts +1 -0
- package/dist/utils/prefixExpansion.js +11 -0
- package/dist/utils/qdrantResponse.d.ts +13 -0
- package/dist/utils/qdrantResponse.js +12 -0
- package/dist/utils/requestIdentity.d.ts +8 -0
- package/dist/utils/requestIdentity.js +52 -0
- package/dist/utils/retry.d.ts +2 -0
- package/dist/utils/retry.js +55 -11
- package/dist/utils/tenant.d.ts +12 -6
- package/dist/utils/tenant.js +41 -32
- package/dist/utils/vectorBinary.d.ts +0 -1
- package/dist/utils/vectorBinary.js +0 -98
- package/dist/utils/ydbErrors.d.ts +1 -0
- package/dist/utils/ydbErrors.js +14 -0
- package/dist/ydb/bootstrapMetaTable.js +14 -2
- package/dist/ydb/client.d.ts +10 -2
- package/dist/ydb/client.js +83 -24
- package/dist/ydb/helpers.d.ts +0 -1
- package/dist/ydb/helpers.js +1 -2
- package/dist/ydb/schema.d.ts +2 -0
- package/dist/ydb/schema.js +84 -7
- package/package.json +10 -5
package/dist/routes/points.js
CHANGED
|
@@ -1,10 +1,37 @@
|
|
|
1
1
|
import { Router } from "express";
|
|
2
|
-
import { upsertPoints, searchPoints, queryPoints, deletePoints, } from "../services/PointsService.js";
|
|
2
|
+
import { upsertPoints, searchPoints, queryPoints, deletePoints, retrievePoints, } from "../services/PointsService.js";
|
|
3
3
|
import { QdrantServiceError } from "../services/errors.js";
|
|
4
4
|
import { logger } from "../logging/logger.js";
|
|
5
5
|
import { isCompilationTimeoutError } from "../ydb/client.js";
|
|
6
6
|
import { scheduleExit } from "../utils/exit.js";
|
|
7
|
+
import { isUpsertRequestTimedOut } from "../middleware/upsertRequestTimeout.js";
|
|
8
|
+
import { qdrantResponse } from "../utils/qdrantResponse.js";
|
|
9
|
+
import { elapsedMsSince, getElapsedMsSinceRequestStart, getMonotonicTimeNs, } from "../logging/requestContext.js";
|
|
10
|
+
import { isAnonymousIdentityError, resolveRequestNamespaceUserUid, resolveRequestSigningKey, } from "../utils/requestIdentity.js";
|
|
7
11
|
export const pointsRouter = Router();
|
|
12
|
+
function buildPointsContext(req) {
|
|
13
|
+
const userUid = resolveRequestNamespaceUserUid(req);
|
|
14
|
+
return {
|
|
15
|
+
userUid,
|
|
16
|
+
collection: String(req.params.collection),
|
|
17
|
+
apiKey: resolveRequestSigningKey(req, userUid),
|
|
18
|
+
userAgent: req.header("User-Agent") ?? undefined,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
function shouldSkipResponse(req, res) {
|
|
22
|
+
return res.headersSent || res.writableEnded || isUpsertRequestTimedOut(req);
|
|
23
|
+
}
|
|
24
|
+
function sendKnownRouteError(res, err) {
|
|
25
|
+
if (err instanceof QdrantServiceError) {
|
|
26
|
+
res.status(err.statusCode).json(err.payload);
|
|
27
|
+
return true;
|
|
28
|
+
}
|
|
29
|
+
if (isAnonymousIdentityError(err)) {
|
|
30
|
+
res.status(400).json({ status: "error", error: err.message });
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
8
35
|
function toQdrantScoredPoint(p) {
|
|
9
36
|
// We don't currently track per-point versions or return vectors/shard keys,
|
|
10
37
|
// but many Qdrant clients expect these fields to exist in the response.
|
|
@@ -18,51 +45,123 @@ function toQdrantScoredPoint(p) {
|
|
|
18
45
|
order_value: null,
|
|
19
46
|
};
|
|
20
47
|
}
|
|
48
|
+
function logUpsertHandlerEntry(alias) {
|
|
49
|
+
logger.info({
|
|
50
|
+
phase: "upsertEntry",
|
|
51
|
+
alias,
|
|
52
|
+
timeToHandlerMs: getElapsedMsSinceRequestStart(),
|
|
53
|
+
}, "upsert: handler entry");
|
|
54
|
+
}
|
|
55
|
+
function logUpsertFailure(args) {
|
|
56
|
+
if (args.err instanceof QdrantServiceError ||
|
|
57
|
+
isAnonymousIdentityError(args.err)) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
logger.error({
|
|
61
|
+
err: args.err instanceof Error
|
|
62
|
+
? args.err
|
|
63
|
+
: new Error(String(args.err)),
|
|
64
|
+
phase: "upsertFailed",
|
|
65
|
+
alias: args.alias,
|
|
66
|
+
routeElapsedMs: elapsedMsSince(args.routeStartNs),
|
|
67
|
+
timeToHandlerMs: getElapsedMsSinceRequestStart(),
|
|
68
|
+
}, "upsert: failed");
|
|
69
|
+
}
|
|
70
|
+
// Qdrant-compatible: POST /collections/:collection/points (retrieve by IDs)
|
|
71
|
+
pointsRouter.post("/:collection/points", async (req, res) => {
|
|
72
|
+
const start = process.hrtime();
|
|
73
|
+
try {
|
|
74
|
+
const { points } = await retrievePoints(buildPointsContext(req), req.body);
|
|
75
|
+
const result = points.map((p) => ({
|
|
76
|
+
id: p.id,
|
|
77
|
+
version: 0,
|
|
78
|
+
payload: p.payload,
|
|
79
|
+
vector: null,
|
|
80
|
+
shard_key: null,
|
|
81
|
+
order_value: null,
|
|
82
|
+
}));
|
|
83
|
+
res.json(qdrantResponse(result, start));
|
|
84
|
+
}
|
|
85
|
+
catch (err) {
|
|
86
|
+
if (sendKnownRouteError(res, err)) {
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
logger.error({ err }, "retrieve points failed");
|
|
90
|
+
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
91
|
+
res.status(500).json({ status: "error", error: errorMessage });
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
// Fix #6: Upsert → UpdateResult shape
|
|
21
95
|
// Qdrant-compatible: PUT /collections/:collection/points (upsert)
|
|
22
96
|
pointsRouter.put("/:collection/points", async (req, res) => {
|
|
97
|
+
const start = process.hrtime();
|
|
98
|
+
const routeStartNs = getMonotonicTimeNs();
|
|
23
99
|
try {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
100
|
+
logUpsertHandlerEntry("put");
|
|
101
|
+
if (shouldSkipResponse(req, res)) {
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
await upsertPoints(buildPointsContext(req), req.body);
|
|
105
|
+
if (shouldSkipResponse(req, res)) {
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
res.json(qdrantResponse({ operation_id: 0, status: "completed" }, start));
|
|
31
109
|
}
|
|
32
110
|
catch (err) {
|
|
33
|
-
|
|
34
|
-
return res.status(err.statusCode).json(err.payload);
|
|
35
|
-
}
|
|
111
|
+
logUpsertFailure({ alias: "put", routeStartNs, err });
|
|
36
112
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
113
|
+
const isCompilationTimeout = isCompilationTimeoutError(err);
|
|
114
|
+
if (isCompilationTimeout) {
|
|
115
|
+
logger.error({ err }, "YDB compilation timeout during upsert points (PUT); scheduling process exit");
|
|
40
116
|
scheduleExit(1);
|
|
117
|
+
if (shouldSkipResponse(req, res)) {
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
res.status(500).json({ status: "error", error: errorMessage });
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
if (shouldSkipResponse(req, res)) {
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
if (sendKnownRouteError(res, err)) {
|
|
41
127
|
return;
|
|
42
128
|
}
|
|
43
129
|
logger.error({ err }, "upsert points (PUT) failed");
|
|
44
130
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
45
131
|
}
|
|
46
132
|
});
|
|
133
|
+
// Fix #6: Upsert → UpdateResult shape
|
|
47
134
|
pointsRouter.post("/:collection/points/upsert", async (req, res) => {
|
|
135
|
+
const start = process.hrtime();
|
|
136
|
+
const routeStartNs = getMonotonicTimeNs();
|
|
48
137
|
try {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
138
|
+
logUpsertHandlerEntry("post");
|
|
139
|
+
if (shouldSkipResponse(req, res)) {
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
await upsertPoints(buildPointsContext(req), req.body);
|
|
143
|
+
if (shouldSkipResponse(req, res)) {
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
res.json(qdrantResponse({ operation_id: 0, status: "completed" }, start));
|
|
56
147
|
}
|
|
57
148
|
catch (err) {
|
|
58
|
-
|
|
59
|
-
return res.status(err.statusCode).json(err.payload);
|
|
60
|
-
}
|
|
149
|
+
logUpsertFailure({ alias: "post", routeStartNs, err });
|
|
61
150
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
151
|
+
const isCompilationTimeout = isCompilationTimeoutError(err);
|
|
152
|
+
if (isCompilationTimeout) {
|
|
153
|
+
logger.error({ err }, "YDB compilation timeout during upsert points; scheduling process exit");
|
|
65
154
|
scheduleExit(1);
|
|
155
|
+
if (shouldSkipResponse(req, res)) {
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
res.status(500).json({ status: "error", error: errorMessage });
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
if (shouldSkipResponse(req, res)) {
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
if (sendKnownRouteError(res, err)) {
|
|
66
165
|
return;
|
|
67
166
|
}
|
|
68
167
|
logger.error({ err }, "upsert points failed");
|
|
@@ -70,22 +169,18 @@ pointsRouter.post("/:collection/points/upsert", async (req, res) => {
|
|
|
70
169
|
}
|
|
71
170
|
});
|
|
72
171
|
pointsRouter.post("/:collection/points/search", async (req, res) => {
|
|
172
|
+
const start = process.hrtime();
|
|
73
173
|
try {
|
|
74
|
-
const { points } = await searchPoints(
|
|
75
|
-
|
|
76
|
-
collection: String(req.params.collection),
|
|
77
|
-
apiKey: req.header("api-key") ?? undefined,
|
|
78
|
-
userAgent: req.header("User-Agent") ?? undefined,
|
|
79
|
-
}, req.body);
|
|
80
|
-
res.json({ status: "ok", result: points.map(toQdrantScoredPoint) });
|
|
174
|
+
const { points } = await searchPoints(buildPointsContext(req), req.body);
|
|
175
|
+
res.json(qdrantResponse(points.map(toQdrantScoredPoint), start));
|
|
81
176
|
}
|
|
82
177
|
catch (err) {
|
|
83
|
-
if (err
|
|
84
|
-
return
|
|
178
|
+
if (sendKnownRouteError(res, err)) {
|
|
179
|
+
return;
|
|
85
180
|
}
|
|
86
181
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
87
182
|
if (isCompilationTimeoutError(err)) {
|
|
88
|
-
logger.error({ err }, "YDB compilation
|
|
183
|
+
logger.error({ err }, "YDB compilation timeout during search points; scheduling process exit");
|
|
89
184
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
90
185
|
scheduleExit(1);
|
|
91
186
|
return;
|
|
@@ -96,22 +191,19 @@ pointsRouter.post("/:collection/points/search", async (req, res) => {
|
|
|
96
191
|
});
|
|
97
192
|
// Compatibility: some clients call POST /collections/:collection/points/query
|
|
98
193
|
pointsRouter.post("/:collection/points/query", async (req, res) => {
|
|
194
|
+
const start = process.hrtime();
|
|
99
195
|
try {
|
|
100
|
-
const { points } = await queryPoints(
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
apiKey: req.header("api-key") ?? undefined,
|
|
104
|
-
userAgent: req.header("User-Agent") ?? undefined,
|
|
105
|
-
}, req.body);
|
|
106
|
-
res.json({ status: "ok", result: points.map(toQdrantScoredPoint) });
|
|
196
|
+
const { points } = await queryPoints(buildPointsContext(req), req.body);
|
|
197
|
+
// Qdrant-compatible: /points/query returns QueryResponse with { points: ScoredPoint[] }.
|
|
198
|
+
res.json(qdrantResponse({ points: points.map(toQdrantScoredPoint) }, start));
|
|
107
199
|
}
|
|
108
200
|
catch (err) {
|
|
109
|
-
if (err
|
|
110
|
-
return
|
|
201
|
+
if (sendKnownRouteError(res, err)) {
|
|
202
|
+
return;
|
|
111
203
|
}
|
|
112
204
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
113
205
|
if (isCompilationTimeoutError(err)) {
|
|
114
|
-
logger.error({ err }, "YDB compilation
|
|
206
|
+
logger.error({ err }, "YDB compilation timeout during search points (query); scheduling process exit");
|
|
115
207
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
116
208
|
scheduleExit(1);
|
|
117
209
|
return;
|
|
@@ -120,19 +212,16 @@ pointsRouter.post("/:collection/points/query", async (req, res) => {
|
|
|
120
212
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
121
213
|
}
|
|
122
214
|
});
|
|
215
|
+
// Fix #7: Delete → UpdateResult shape
|
|
123
216
|
pointsRouter.post("/:collection/points/delete", async (req, res) => {
|
|
217
|
+
const start = process.hrtime();
|
|
124
218
|
try {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
collection: String(req.params.collection),
|
|
128
|
-
apiKey: req.header("api-key") ?? undefined,
|
|
129
|
-
userAgent: req.header("User-Agent") ?? undefined,
|
|
130
|
-
}, req.body);
|
|
131
|
-
res.json({ status: "ok", result });
|
|
219
|
+
await deletePoints(buildPointsContext(req), req.body);
|
|
220
|
+
res.json(qdrantResponse({ operation_id: 0, status: "completed" }, start));
|
|
132
221
|
}
|
|
133
222
|
catch (err) {
|
|
134
|
-
if (err
|
|
135
|
-
return
|
|
223
|
+
if (sendKnownRouteError(res, err)) {
|
|
224
|
+
return;
|
|
136
225
|
}
|
|
137
226
|
logger.error({ err }, "delete points failed");
|
|
138
227
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
package/dist/server.js
CHANGED
|
@@ -2,6 +2,8 @@ import express from "express";
|
|
|
2
2
|
import { collectionsRouter } from "./routes/collections.js";
|
|
3
3
|
import { pointsRouter } from "./routes/points.js";
|
|
4
4
|
import { requestLogger } from "./middleware/requestLogger.js";
|
|
5
|
+
import { isUpsertRequestTimedOut, upsertBodyTimeout, upsertProcessingTimeout, } from "./middleware/upsertRequestTimeout.js";
|
|
6
|
+
import { instrumentUpsertRequestBody, logUpsertBodyPhase, logUpsertBodyPhaseError, verifyUpsertRequestBody, } from "./middleware/upsertBodyPhase.js";
|
|
5
7
|
import { isYdbAvailable, isCompilationTimeoutError } from "./ydb/client.js";
|
|
6
8
|
import { verifyCollectionsQueryCompilationForStartup } from "./repositories/collectionsRepo.js";
|
|
7
9
|
import { logger } from "./logging/logger.js";
|
|
@@ -9,7 +11,7 @@ import { scheduleExit } from "./utils/exit.js";
|
|
|
9
11
|
export async function healthHandler(_req, res) {
|
|
10
12
|
const ok = await isYdbAvailable();
|
|
11
13
|
if (!ok) {
|
|
12
|
-
logger.error("YDB unavailable during health check
|
|
14
|
+
logger.error("YDB unavailable during health check");
|
|
13
15
|
res.status(503).json({ status: "error", error: "YDB unavailable" });
|
|
14
16
|
scheduleExit(1);
|
|
15
17
|
return;
|
|
@@ -20,8 +22,8 @@ export async function healthHandler(_req, res) {
|
|
|
20
22
|
catch (err) {
|
|
21
23
|
const isTimeout = isCompilationTimeoutError(err);
|
|
22
24
|
logger.error({ err }, isTimeout
|
|
23
|
-
? "YDB compilation timeout during health probe
|
|
24
|
-
: "YDB health probe failed
|
|
25
|
+
? "YDB compilation timeout during health probe"
|
|
26
|
+
: "YDB health probe failed");
|
|
25
27
|
res.status(503).json({
|
|
26
28
|
status: "error",
|
|
27
29
|
error: "YDB health probe failed",
|
|
@@ -38,7 +40,15 @@ export function rootHandler(_req, res) {
|
|
|
38
40
|
export function buildServer() {
|
|
39
41
|
const app = express();
|
|
40
42
|
app.use(requestLogger);
|
|
41
|
-
app.use(
|
|
43
|
+
app.use(upsertBodyTimeout);
|
|
44
|
+
app.use(instrumentUpsertRequestBody);
|
|
45
|
+
app.use(express.json({
|
|
46
|
+
limit: "20mb",
|
|
47
|
+
verify: verifyUpsertRequestBody,
|
|
48
|
+
}));
|
|
49
|
+
app.use(logUpsertBodyPhaseError);
|
|
50
|
+
app.use(logUpsertBodyPhase);
|
|
51
|
+
app.use(upsertProcessingTimeout);
|
|
42
52
|
app.get("/", rootHandler);
|
|
43
53
|
app.get("/health", healthHandler);
|
|
44
54
|
app.use("/collections", collectionsRouter);
|
|
@@ -57,12 +67,16 @@ export function buildServer() {
|
|
|
57
67
|
});
|
|
58
68
|
// Catch-all error handler: avoid Express default handler printing stacktraces to stderr
|
|
59
69
|
// and provide consistent JSON error responses.
|
|
60
|
-
app.use((err,
|
|
61
|
-
logger.error({ err }, "Unhandled error in Express middleware");
|
|
70
|
+
app.use((err, req, res, _next) => {
|
|
62
71
|
void _next;
|
|
63
72
|
if (res.headersSent || res.writableEnded) {
|
|
73
|
+
if (shouldSuppressLateMiddlewareError(req, err)) {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
logger.error({ err }, "Late middleware error after response sent");
|
|
64
77
|
return;
|
|
65
78
|
}
|
|
79
|
+
logger.error({ err }, "Unhandled error in Express middleware");
|
|
66
80
|
const statusCode = extractHttpStatusCode(err) ?? 500;
|
|
67
81
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
68
82
|
res.status(statusCode).json({
|
|
@@ -85,6 +99,28 @@ function isRequestAbortedError(err) {
|
|
|
85
99
|
}
|
|
86
100
|
return false;
|
|
87
101
|
}
|
|
102
|
+
function isBodyParserRequestReadError(err) {
|
|
103
|
+
if (!err || typeof err !== "object") {
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
const typeValue = "type" in err && typeof err.type === "string" ? err.type : undefined;
|
|
107
|
+
if (!typeValue) {
|
|
108
|
+
return false;
|
|
109
|
+
}
|
|
110
|
+
return (typeValue === "entity.parse.failed" ||
|
|
111
|
+
typeValue === "entity.too.large" ||
|
|
112
|
+
typeValue === "request.size.invalid" ||
|
|
113
|
+
typeValue === "charset.unsupported" ||
|
|
114
|
+
typeValue === "encoding.unsupported" ||
|
|
115
|
+
typeValue === "stream.encoding.set" ||
|
|
116
|
+
typeValue === "stream.not.readable");
|
|
117
|
+
}
|
|
118
|
+
function shouldSuppressLateMiddlewareError(req, err) {
|
|
119
|
+
if (isRequestAbortedError(err)) {
|
|
120
|
+
return true;
|
|
121
|
+
}
|
|
122
|
+
return isUpsertRequestTimedOut(req) && isBodyParserRequestReadError(err);
|
|
123
|
+
}
|
|
88
124
|
function extractHttpStatusCode(err) {
|
|
89
125
|
if (!err || typeof err !== "object") {
|
|
90
126
|
return undefined;
|
|
@@ -1,23 +1,25 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { DistanceKind } from "../qdrant/QdrantRestTypes.js";
|
|
2
2
|
export interface CollectionContextInput {
|
|
3
|
-
|
|
3
|
+
userUid: string;
|
|
4
4
|
collection: string;
|
|
5
|
-
apiKey
|
|
5
|
+
apiKey: string;
|
|
6
6
|
userAgent?: string;
|
|
7
7
|
}
|
|
8
8
|
export interface NormalizedCollectionContext {
|
|
9
|
-
|
|
9
|
+
userUid: string;
|
|
10
10
|
collection: string;
|
|
11
11
|
metaKey: string;
|
|
12
|
+
uid: string;
|
|
12
13
|
}
|
|
13
14
|
export declare function putCollectionIndex(ctx: CollectionContextInput): Promise<{
|
|
14
15
|
acknowledged: boolean;
|
|
15
16
|
}>;
|
|
16
17
|
export declare function createCollection(ctx: CollectionContextInput, body: unknown): Promise<{
|
|
17
18
|
name: string;
|
|
18
|
-
tenant: string;
|
|
19
19
|
}>;
|
|
20
20
|
export declare function getCollection(ctx: CollectionContextInput): Promise<{
|
|
21
|
+
status: string;
|
|
22
|
+
points_count: number;
|
|
21
23
|
name: string;
|
|
22
24
|
vectors: {
|
|
23
25
|
size: number;
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { CreateCollectionReq } from "../
|
|
1
|
+
import { CreateCollectionReq } from "../qdrant/Requests.js";
|
|
2
2
|
import { ensureMetaTable } from "../ydb/schema.js";
|
|
3
|
-
import { createCollection as repoCreateCollection, deleteCollection as repoDeleteCollection, getCollectionMeta, touchCollectionLastAccess, } from "../repositories/collectionsRepo.js";
|
|
3
|
+
import { createCollection as repoCreateCollection, countPointsForCollection, deleteCollection as repoDeleteCollection, getCollectionMeta, touchCollectionLastAccess, } from "../repositories/collectionsRepo.js";
|
|
4
4
|
import { QdrantServiceError } from "./errors.js";
|
|
5
5
|
import { normalizeCollectionContextShared } from "./CollectionService.shared.js";
|
|
6
6
|
export async function putCollectionIndex(ctx) {
|
|
7
7
|
await ensureMetaTable();
|
|
8
|
-
const normalized = normalizeCollectionContextShared(ctx.
|
|
8
|
+
const normalized = normalizeCollectionContextShared(ctx.userUid, ctx.collection, ctx.userAgent);
|
|
9
9
|
const meta = await getCollectionMeta(normalized.metaKey);
|
|
10
10
|
if (!meta) {
|
|
11
11
|
throw new QdrantServiceError(404, {
|
|
@@ -18,7 +18,7 @@ export async function putCollectionIndex(ctx) {
|
|
|
18
18
|
}
|
|
19
19
|
export async function createCollection(ctx, body) {
|
|
20
20
|
await ensureMetaTable();
|
|
21
|
-
const normalized = normalizeCollectionContextShared(ctx.
|
|
21
|
+
const normalized = normalizeCollectionContextShared(ctx.userUid, ctx.collection, ctx.userAgent);
|
|
22
22
|
const parsed = CreateCollectionReq.safeParse(body);
|
|
23
23
|
if (!parsed.success) {
|
|
24
24
|
throw new QdrantServiceError(400, {
|
|
@@ -35,7 +35,7 @@ export async function createCollection(ctx, body) {
|
|
|
35
35
|
existing.distance === distance &&
|
|
36
36
|
existing.vectorType === vectorType) {
|
|
37
37
|
await touchCollectionLastAccess(normalized.metaKey);
|
|
38
|
-
return { name: normalized.collection
|
|
38
|
+
return { name: normalized.collection };
|
|
39
39
|
}
|
|
40
40
|
const errorMessage = `Collection already exists with different config: dimension=${existing.dimension}, distance=${existing.distance}, type=${existing.vectorType}`;
|
|
41
41
|
throw new QdrantServiceError(400, {
|
|
@@ -43,12 +43,12 @@ export async function createCollection(ctx, body) {
|
|
|
43
43
|
error: errorMessage,
|
|
44
44
|
});
|
|
45
45
|
}
|
|
46
|
-
await repoCreateCollection(normalized.metaKey, dim, distance, vectorType);
|
|
47
|
-
return { name: normalized.collection
|
|
46
|
+
await repoCreateCollection(normalized.metaKey, dim, distance, vectorType, ctx.userUid);
|
|
47
|
+
return { name: normalized.collection };
|
|
48
48
|
}
|
|
49
49
|
export async function getCollection(ctx) {
|
|
50
50
|
await ensureMetaTable();
|
|
51
|
-
const normalized = normalizeCollectionContextShared(ctx.
|
|
51
|
+
const normalized = normalizeCollectionContextShared(ctx.userUid, ctx.collection, ctx.userAgent);
|
|
52
52
|
const meta = await getCollectionMeta(normalized.metaKey);
|
|
53
53
|
if (!meta) {
|
|
54
54
|
throw new QdrantServiceError(404, {
|
|
@@ -56,8 +56,11 @@ export async function getCollection(ctx) {
|
|
|
56
56
|
error: "collection not found",
|
|
57
57
|
});
|
|
58
58
|
}
|
|
59
|
+
const pointsCount = await countPointsForCollection(normalized.uid);
|
|
59
60
|
await touchCollectionLastAccess(normalized.metaKey);
|
|
60
61
|
return {
|
|
62
|
+
status: "green",
|
|
63
|
+
points_count: pointsCount,
|
|
61
64
|
name: normalized.collection,
|
|
62
65
|
vectors: {
|
|
63
66
|
size: meta.dimension,
|
|
@@ -77,7 +80,7 @@ export async function getCollection(ctx) {
|
|
|
77
80
|
}
|
|
78
81
|
export async function deleteCollection(ctx) {
|
|
79
82
|
await ensureMetaTable();
|
|
80
|
-
const normalized = normalizeCollectionContextShared(ctx.
|
|
83
|
+
const normalized = normalizeCollectionContextShared(ctx.userUid, ctx.collection, ctx.userAgent);
|
|
81
84
|
await repoDeleteCollection(normalized.metaKey);
|
|
82
85
|
return { acknowledged: true };
|
|
83
86
|
}
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import { uidFor, } from "./CollectionService.shared.js";
|
|
2
1
|
import { GLOBAL_POINTS_TABLE, ensureGlobalPointsTable } from "../ydb/schema.js";
|
|
3
2
|
export async function resolvePointsTableAndUidOneTable(ctx) {
|
|
4
3
|
await ensureGlobalPointsTable();
|
|
5
4
|
return {
|
|
6
5
|
tableName: GLOBAL_POINTS_TABLE,
|
|
7
|
-
uid:
|
|
6
|
+
uid: ctx.uid,
|
|
8
7
|
};
|
|
9
8
|
}
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
export interface NormalizedCollectionContextLike {
|
|
2
|
-
|
|
2
|
+
userUid: string;
|
|
3
3
|
collection: string;
|
|
4
4
|
metaKey: string;
|
|
5
|
+
uid: string;
|
|
5
6
|
}
|
|
6
|
-
export declare function
|
|
7
|
-
export declare function
|
|
8
|
-
|
|
9
|
-
tenant: string;
|
|
7
|
+
export declare function uidForCollection(userUid: string, collection: string): string;
|
|
8
|
+
export declare function normalizeCollectionContextShared(userUid: string, collection: string, _userAgent?: string): {
|
|
9
|
+
userUid: string;
|
|
10
10
|
collection: string;
|
|
11
11
|
metaKey: string;
|
|
12
|
+
uid: string;
|
|
12
13
|
};
|
|
@@ -1,19 +1,35 @@
|
|
|
1
|
-
import { sanitizeCollectionName,
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
import { sanitizeCollectionName, metaKeyFor, uidFor, } from "../utils/tenant.js";
|
|
2
|
+
import { QdrantServiceError } from "./errors.js";
|
|
3
|
+
function requireAndSanitizeUserUid(userUid) {
|
|
4
|
+
if (!userUid || userUid.trim() === "") {
|
|
5
|
+
throw new QdrantServiceError(401, {
|
|
6
|
+
status: "error",
|
|
7
|
+
error: "unauthorized",
|
|
8
|
+
});
|
|
9
|
+
}
|
|
10
|
+
const raw = userUid.toString();
|
|
11
|
+
const cleaned = raw.replace(/[^a-zA-Z0-9_]/g, "_").replace(/_+/g, "_");
|
|
12
|
+
const lowered = cleaned.toLowerCase().replace(/^_+/, "");
|
|
13
|
+
if (lowered.length === 0) {
|
|
14
|
+
throw new QdrantServiceError(401, {
|
|
15
|
+
status: "error",
|
|
16
|
+
error: "unauthorized",
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
return lowered;
|
|
4
20
|
}
|
|
5
|
-
export function
|
|
6
|
-
return
|
|
21
|
+
export function uidForCollection(userUid, collection) {
|
|
22
|
+
return uidFor(userUid, collection);
|
|
7
23
|
}
|
|
8
|
-
export function normalizeCollectionContextShared(
|
|
9
|
-
|
|
10
|
-
const
|
|
11
|
-
const
|
|
12
|
-
const
|
|
13
|
-
const metaKey = metaKeyFor(normalizedTenant, normalizedCollection);
|
|
24
|
+
export function normalizeCollectionContextShared(userUid, collection, _userAgent) {
|
|
25
|
+
void _userAgent;
|
|
26
|
+
const normalizedUserUid = requireAndSanitizeUserUid(userUid);
|
|
27
|
+
const normalizedCollection = sanitizeCollectionName(collection);
|
|
28
|
+
const metaKey = metaKeyFor(normalizedUserUid, normalizedCollection);
|
|
14
29
|
return {
|
|
15
|
-
|
|
30
|
+
userUid: normalizedUserUid,
|
|
16
31
|
collection: normalizedCollection,
|
|
17
32
|
metaKey,
|
|
33
|
+
uid: uidForCollection(normalizedUserUid, normalizedCollection),
|
|
18
34
|
};
|
|
19
35
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type CollectionContextInput } from "./CollectionService.js";
|
|
2
2
|
import type { YdbQdrantScoredPoint } from "../qdrant/QdrantRestTypes.js";
|
|
3
|
+
import type { RetrievedPoint } from "../repositories/pointsRepo.one-table/Retrieve.js";
|
|
3
4
|
type PointsContextInput = CollectionContextInput;
|
|
4
5
|
export declare function upsertPoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
5
6
|
upserted: number;
|
|
@@ -10,6 +11,13 @@ export declare function searchPoints(ctx: PointsContextInput, body: unknown): Pr
|
|
|
10
11
|
export declare function queryPoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
11
12
|
points: YdbQdrantScoredPoint[];
|
|
12
13
|
}>;
|
|
14
|
+
export declare function retrievePoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
15
|
+
points: RetrievedPoint[];
|
|
16
|
+
}>;
|
|
17
|
+
/**
|
|
18
|
+
* @returns `deleted` — number of points removed, or `-1` when the exact count
|
|
19
|
+
* is unavailable (e.g. bulk BATCH DELETE or clear-all-points).
|
|
20
|
+
*/
|
|
13
21
|
export declare function deletePoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
14
22
|
deleted: number;
|
|
15
23
|
}>;
|