ydb-qdrant 8.1.1 → 9.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -18
- package/dist/SmokeTest.js +2 -2
- package/dist/compute/ComputePool.d.ts +5 -0
- package/dist/compute/ComputePool.js +64 -0
- package/dist/compute/ComputeWorker.d.ts +36 -0
- package/dist/compute/ComputeWorker.js +84 -0
- package/dist/config/env.d.ts +24 -7
- package/dist/config/env.js +65 -35
- package/dist/index.d.ts +2 -0
- package/dist/index.js +92 -2
- package/dist/logging/DeployLogFormatter.d.ts +2 -0
- package/dist/logging/DeployLogFormatter.js +131 -0
- package/dist/logging/logger.js +13 -1
- package/dist/logging/requestContext.d.ts +17 -0
- package/dist/logging/requestContext.js +43 -0
- package/dist/middleware/requestLogger.js +134 -6
- package/dist/middleware/upsertBodyPhase.d.ts +6 -0
- package/dist/middleware/upsertBodyPhase.js +184 -0
- package/dist/middleware/upsertRequestTimeout.d.ts +16 -0
- package/dist/middleware/upsertRequestTimeout.js +158 -0
- package/dist/package/api.d.ts +20 -12
- package/dist/package/api.js +57 -28
- package/dist/qdrant/QdrantRestTypes.d.ts +4 -0
- package/dist/qdrant/Requests.d.ts +97 -0
- package/dist/qdrant/Requests.js +72 -0
- package/dist/repositories/collectionsRepo.d.ts +18 -2
- package/dist/repositories/collectionsRepo.js +103 -7
- package/dist/repositories/collectionsRepo.one-table.d.ts +4 -3
- package/dist/repositories/collectionsRepo.one-table.js +99 -36
- package/dist/repositories/collectionsRepo.shared.d.ts +2 -2
- package/dist/repositories/collectionsRepo.shared.js +9 -4
- package/dist/repositories/pointsRepo.d.ts +6 -4
- package/dist/repositories/pointsRepo.js +8 -7
- package/dist/repositories/pointsRepo.one-table/Delete.d.ts +2 -2
- package/dist/repositories/pointsRepo.one-table/Delete.js +157 -60
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.d.ts +7 -5
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.js +44 -13
- package/dist/repositories/pointsRepo.one-table/Retrieve.d.ts +6 -0
- package/dist/repositories/pointsRepo.one-table/Retrieve.js +69 -0
- package/dist/repositories/pointsRepo.one-table/Search.d.ts +2 -3
- package/dist/repositories/pointsRepo.one-table/Search.js +102 -124
- package/dist/repositories/pointsRepo.one-table/Upsert.d.ts +2 -2
- package/dist/repositories/pointsRepo.one-table/Upsert.js +244 -48
- package/dist/repositories/pointsRepo.one-table.d.ts +1 -0
- package/dist/repositories/pointsRepo.one-table.js +1 -0
- package/dist/routes/collections.js +45 -36
- package/dist/routes/points.js +144 -59
- package/dist/server.js +42 -6
- package/dist/services/CollectionService.d.ts +7 -5
- package/dist/services/CollectionService.js +12 -9
- package/dist/services/CollectionService.one-table.js +1 -2
- package/dist/services/CollectionService.shared.d.ts +6 -5
- package/dist/services/CollectionService.shared.js +28 -12
- package/dist/services/PointsService.d.ts +8 -0
- package/dist/services/PointsService.js +132 -15
- package/dist/types.d.ts +4 -94
- package/dist/types.js +1 -54
- package/dist/utils/EnvParsers.d.ts +5 -0
- package/dist/utils/EnvParsers.js +30 -0
- package/dist/utils/PayloadSign.d.ts +4 -0
- package/dist/utils/PayloadSign.js +18 -0
- package/dist/utils/distance.d.ts +1 -12
- package/dist/utils/distance.js +0 -21
- package/dist/utils/pathPrefix.d.ts +3 -0
- package/dist/utils/pathPrefix.js +47 -0
- package/dist/utils/prefixExpansion.d.ts +1 -0
- package/dist/utils/prefixExpansion.js +11 -0
- package/dist/utils/qdrantResponse.d.ts +13 -0
- package/dist/utils/qdrantResponse.js +12 -0
- package/dist/utils/requestIdentity.d.ts +8 -0
- package/dist/utils/requestIdentity.js +52 -0
- package/dist/utils/retry.d.ts +2 -0
- package/dist/utils/retry.js +55 -11
- package/dist/utils/tenant.d.ts +12 -6
- package/dist/utils/tenant.js +41 -32
- package/dist/utils/vectorBinary.d.ts +0 -1
- package/dist/utils/vectorBinary.js +0 -98
- package/dist/utils/ydbErrors.d.ts +1 -0
- package/dist/utils/ydbErrors.js +14 -0
- package/dist/ydb/bootstrapMetaTable.js +14 -2
- package/dist/ydb/client.d.ts +10 -2
- package/dist/ydb/client.js +83 -24
- package/dist/ydb/helpers.d.ts +0 -1
- package/dist/ydb/helpers.js +1 -2
- package/dist/ydb/schema.d.ts +2 -0
- package/dist/ydb/schema.js +84 -7
- package/package.json +10 -5
package/dist/routes/points.js
CHANGED
|
@@ -1,10 +1,37 @@
|
|
|
1
1
|
import { Router } from "express";
|
|
2
|
-
import { upsertPoints, searchPoints, queryPoints, deletePoints, } from "../services/PointsService.js";
|
|
2
|
+
import { upsertPoints, searchPoints, queryPoints, deletePoints, retrievePoints, } from "../services/PointsService.js";
|
|
3
3
|
import { QdrantServiceError } from "../services/errors.js";
|
|
4
4
|
import { logger } from "../logging/logger.js";
|
|
5
5
|
import { isCompilationTimeoutError } from "../ydb/client.js";
|
|
6
6
|
import { scheduleExit } from "../utils/exit.js";
|
|
7
|
+
import { isUpsertRequestTimedOut } from "../middleware/upsertRequestTimeout.js";
|
|
8
|
+
import { qdrantResponse } from "../utils/qdrantResponse.js";
|
|
9
|
+
import { elapsedMsSince, getElapsedMsSinceRequestStart, getMonotonicTimeNs, } from "../logging/requestContext.js";
|
|
10
|
+
import { isAnonymousIdentityError, resolveRequestNamespaceUserUid, resolveRequestSigningKey, } from "../utils/requestIdentity.js";
|
|
7
11
|
export const pointsRouter = Router();
|
|
12
|
+
function buildPointsContext(req) {
|
|
13
|
+
const userUid = resolveRequestNamespaceUserUid(req);
|
|
14
|
+
return {
|
|
15
|
+
userUid,
|
|
16
|
+
collection: String(req.params.collection),
|
|
17
|
+
apiKey: resolveRequestSigningKey(req, userUid),
|
|
18
|
+
userAgent: req.header("User-Agent") ?? undefined,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
function shouldSkipResponse(req, res) {
|
|
22
|
+
return res.headersSent || res.writableEnded || isUpsertRequestTimedOut(req);
|
|
23
|
+
}
|
|
24
|
+
function sendKnownRouteError(res, err) {
|
|
25
|
+
if (err instanceof QdrantServiceError) {
|
|
26
|
+
res.status(err.statusCode).json(err.payload);
|
|
27
|
+
return true;
|
|
28
|
+
}
|
|
29
|
+
if (isAnonymousIdentityError(err)) {
|
|
30
|
+
res.status(400).json({ status: "error", error: err.message });
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
8
35
|
function toQdrantScoredPoint(p) {
|
|
9
36
|
// We don't currently track per-point versions or return vectors/shard keys,
|
|
10
37
|
// but many Qdrant clients expect these fields to exist in the response.
|
|
@@ -18,51 +45,123 @@ function toQdrantScoredPoint(p) {
|
|
|
18
45
|
order_value: null,
|
|
19
46
|
};
|
|
20
47
|
}
|
|
48
|
+
function logUpsertHandlerEntry(alias) {
|
|
49
|
+
logger.info({
|
|
50
|
+
phase: "upsertEntry",
|
|
51
|
+
alias,
|
|
52
|
+
timeToHandlerMs: getElapsedMsSinceRequestStart(),
|
|
53
|
+
}, "upsert: handler entry");
|
|
54
|
+
}
|
|
55
|
+
function logUpsertFailure(args) {
|
|
56
|
+
if (args.err instanceof QdrantServiceError ||
|
|
57
|
+
isAnonymousIdentityError(args.err)) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
logger.error({
|
|
61
|
+
err: args.err instanceof Error
|
|
62
|
+
? args.err
|
|
63
|
+
: new Error(String(args.err)),
|
|
64
|
+
phase: "upsertFailed",
|
|
65
|
+
alias: args.alias,
|
|
66
|
+
routeElapsedMs: elapsedMsSince(args.routeStartNs),
|
|
67
|
+
timeToHandlerMs: getElapsedMsSinceRequestStart(),
|
|
68
|
+
}, "upsert: failed");
|
|
69
|
+
}
|
|
70
|
+
// Qdrant-compatible: POST /collections/:collection/points (retrieve by IDs)
|
|
71
|
+
pointsRouter.post("/:collection/points", async (req, res) => {
|
|
72
|
+
const start = process.hrtime();
|
|
73
|
+
try {
|
|
74
|
+
const { points } = await retrievePoints(buildPointsContext(req), req.body);
|
|
75
|
+
const result = points.map((p) => ({
|
|
76
|
+
id: p.id,
|
|
77
|
+
version: 0,
|
|
78
|
+
payload: p.payload,
|
|
79
|
+
vector: null,
|
|
80
|
+
shard_key: null,
|
|
81
|
+
order_value: null,
|
|
82
|
+
}));
|
|
83
|
+
res.json(qdrantResponse(result, start));
|
|
84
|
+
}
|
|
85
|
+
catch (err) {
|
|
86
|
+
if (sendKnownRouteError(res, err)) {
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
logger.error({ err }, "retrieve points failed");
|
|
90
|
+
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
91
|
+
res.status(500).json({ status: "error", error: errorMessage });
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
// Fix #6: Upsert → UpdateResult shape
|
|
21
95
|
// Qdrant-compatible: PUT /collections/:collection/points (upsert)
|
|
22
96
|
pointsRouter.put("/:collection/points", async (req, res) => {
|
|
97
|
+
const start = process.hrtime();
|
|
98
|
+
const routeStartNs = getMonotonicTimeNs();
|
|
23
99
|
try {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
100
|
+
logUpsertHandlerEntry("put");
|
|
101
|
+
if (shouldSkipResponse(req, res)) {
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
await upsertPoints(buildPointsContext(req), req.body);
|
|
105
|
+
if (shouldSkipResponse(req, res)) {
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
res.json(qdrantResponse({ operation_id: 0, status: "completed" }, start));
|
|
31
109
|
}
|
|
32
110
|
catch (err) {
|
|
33
|
-
|
|
34
|
-
return res.status(err.statusCode).json(err.payload);
|
|
35
|
-
}
|
|
111
|
+
logUpsertFailure({ alias: "put", routeStartNs, err });
|
|
36
112
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
113
|
+
const isCompilationTimeout = isCompilationTimeoutError(err);
|
|
114
|
+
if (isCompilationTimeout) {
|
|
115
|
+
logger.error({ err }, "YDB compilation timeout during upsert points (PUT); scheduling process exit");
|
|
40
116
|
scheduleExit(1);
|
|
117
|
+
if (shouldSkipResponse(req, res)) {
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
res.status(500).json({ status: "error", error: errorMessage });
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
if (shouldSkipResponse(req, res)) {
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
if (sendKnownRouteError(res, err)) {
|
|
41
127
|
return;
|
|
42
128
|
}
|
|
43
129
|
logger.error({ err }, "upsert points (PUT) failed");
|
|
44
130
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
45
131
|
}
|
|
46
132
|
});
|
|
133
|
+
// Fix #6: Upsert → UpdateResult shape
|
|
47
134
|
pointsRouter.post("/:collection/points/upsert", async (req, res) => {
|
|
135
|
+
const start = process.hrtime();
|
|
136
|
+
const routeStartNs = getMonotonicTimeNs();
|
|
48
137
|
try {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
138
|
+
logUpsertHandlerEntry("post");
|
|
139
|
+
if (shouldSkipResponse(req, res)) {
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
await upsertPoints(buildPointsContext(req), req.body);
|
|
143
|
+
if (shouldSkipResponse(req, res)) {
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
res.json(qdrantResponse({ operation_id: 0, status: "completed" }, start));
|
|
56
147
|
}
|
|
57
148
|
catch (err) {
|
|
58
|
-
|
|
59
|
-
return res.status(err.statusCode).json(err.payload);
|
|
60
|
-
}
|
|
149
|
+
logUpsertFailure({ alias: "post", routeStartNs, err });
|
|
61
150
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
151
|
+
const isCompilationTimeout = isCompilationTimeoutError(err);
|
|
152
|
+
if (isCompilationTimeout) {
|
|
153
|
+
logger.error({ err }, "YDB compilation timeout during upsert points; scheduling process exit");
|
|
65
154
|
scheduleExit(1);
|
|
155
|
+
if (shouldSkipResponse(req, res)) {
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
res.status(500).json({ status: "error", error: errorMessage });
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
if (shouldSkipResponse(req, res)) {
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
if (sendKnownRouteError(res, err)) {
|
|
66
165
|
return;
|
|
67
166
|
}
|
|
68
167
|
logger.error({ err }, "upsert points failed");
|
|
@@ -70,22 +169,18 @@ pointsRouter.post("/:collection/points/upsert", async (req, res) => {
|
|
|
70
169
|
}
|
|
71
170
|
});
|
|
72
171
|
pointsRouter.post("/:collection/points/search", async (req, res) => {
|
|
172
|
+
const start = process.hrtime();
|
|
73
173
|
try {
|
|
74
|
-
const { points } = await searchPoints(
|
|
75
|
-
|
|
76
|
-
collection: String(req.params.collection),
|
|
77
|
-
apiKey: req.header("api-key") ?? undefined,
|
|
78
|
-
userAgent: req.header("User-Agent") ?? undefined,
|
|
79
|
-
}, req.body);
|
|
80
|
-
res.json({ status: "ok", result: points.map(toQdrantScoredPoint) });
|
|
174
|
+
const { points } = await searchPoints(buildPointsContext(req), req.body);
|
|
175
|
+
res.json(qdrantResponse(points.map(toQdrantScoredPoint), start));
|
|
81
176
|
}
|
|
82
177
|
catch (err) {
|
|
83
|
-
if (err
|
|
84
|
-
return
|
|
178
|
+
if (sendKnownRouteError(res, err)) {
|
|
179
|
+
return;
|
|
85
180
|
}
|
|
86
181
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
87
182
|
if (isCompilationTimeoutError(err)) {
|
|
88
|
-
logger.error({ err }, "YDB compilation
|
|
183
|
+
logger.error({ err }, "YDB compilation timeout during search points; scheduling process exit");
|
|
89
184
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
90
185
|
scheduleExit(1);
|
|
91
186
|
return;
|
|
@@ -96,26 +191,19 @@ pointsRouter.post("/:collection/points/search", async (req, res) => {
|
|
|
96
191
|
});
|
|
97
192
|
// Compatibility: some clients call POST /collections/:collection/points/query
|
|
98
193
|
pointsRouter.post("/:collection/points/query", async (req, res) => {
|
|
194
|
+
const start = process.hrtime();
|
|
99
195
|
try {
|
|
100
|
-
const { points } = await queryPoints(
|
|
101
|
-
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
102
|
-
collection: String(req.params.collection),
|
|
103
|
-
apiKey: req.header("api-key") ?? undefined,
|
|
104
|
-
userAgent: req.header("User-Agent") ?? undefined,
|
|
105
|
-
}, req.body);
|
|
196
|
+
const { points } = await queryPoints(buildPointsContext(req), req.body);
|
|
106
197
|
// Qdrant-compatible: /points/query returns QueryResponse with { points: ScoredPoint[] }.
|
|
107
|
-
res.json({
|
|
108
|
-
status: "ok",
|
|
109
|
-
result: { points: points.map(toQdrantScoredPoint) },
|
|
110
|
-
});
|
|
198
|
+
res.json(qdrantResponse({ points: points.map(toQdrantScoredPoint) }, start));
|
|
111
199
|
}
|
|
112
200
|
catch (err) {
|
|
113
|
-
if (err
|
|
114
|
-
return
|
|
201
|
+
if (sendKnownRouteError(res, err)) {
|
|
202
|
+
return;
|
|
115
203
|
}
|
|
116
204
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
117
205
|
if (isCompilationTimeoutError(err)) {
|
|
118
|
-
logger.error({ err }, "YDB compilation
|
|
206
|
+
logger.error({ err }, "YDB compilation timeout during search points (query); scheduling process exit");
|
|
119
207
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
120
208
|
scheduleExit(1);
|
|
121
209
|
return;
|
|
@@ -124,19 +212,16 @@ pointsRouter.post("/:collection/points/query", async (req, res) => {
|
|
|
124
212
|
res.status(500).json({ status: "error", error: errorMessage });
|
|
125
213
|
}
|
|
126
214
|
});
|
|
215
|
+
// Fix #7: Delete → UpdateResult shape
|
|
127
216
|
pointsRouter.post("/:collection/points/delete", async (req, res) => {
|
|
217
|
+
const start = process.hrtime();
|
|
128
218
|
try {
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
collection: String(req.params.collection),
|
|
132
|
-
apiKey: req.header("api-key") ?? undefined,
|
|
133
|
-
userAgent: req.header("User-Agent") ?? undefined,
|
|
134
|
-
}, req.body);
|
|
135
|
-
res.json({ status: "ok", result });
|
|
219
|
+
await deletePoints(buildPointsContext(req), req.body);
|
|
220
|
+
res.json(qdrantResponse({ operation_id: 0, status: "completed" }, start));
|
|
136
221
|
}
|
|
137
222
|
catch (err) {
|
|
138
|
-
if (err
|
|
139
|
-
return
|
|
223
|
+
if (sendKnownRouteError(res, err)) {
|
|
224
|
+
return;
|
|
140
225
|
}
|
|
141
226
|
logger.error({ err }, "delete points failed");
|
|
142
227
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
package/dist/server.js
CHANGED
|
@@ -2,6 +2,8 @@ import express from "express";
|
|
|
2
2
|
import { collectionsRouter } from "./routes/collections.js";
|
|
3
3
|
import { pointsRouter } from "./routes/points.js";
|
|
4
4
|
import { requestLogger } from "./middleware/requestLogger.js";
|
|
5
|
+
import { isUpsertRequestTimedOut, upsertBodyTimeout, upsertProcessingTimeout, } from "./middleware/upsertRequestTimeout.js";
|
|
6
|
+
import { instrumentUpsertRequestBody, logUpsertBodyPhase, logUpsertBodyPhaseError, verifyUpsertRequestBody, } from "./middleware/upsertBodyPhase.js";
|
|
5
7
|
import { isYdbAvailable, isCompilationTimeoutError } from "./ydb/client.js";
|
|
6
8
|
import { verifyCollectionsQueryCompilationForStartup } from "./repositories/collectionsRepo.js";
|
|
7
9
|
import { logger } from "./logging/logger.js";
|
|
@@ -9,7 +11,7 @@ import { scheduleExit } from "./utils/exit.js";
|
|
|
9
11
|
export async function healthHandler(_req, res) {
|
|
10
12
|
const ok = await isYdbAvailable();
|
|
11
13
|
if (!ok) {
|
|
12
|
-
logger.error("YDB unavailable during health check
|
|
14
|
+
logger.error("YDB unavailable during health check");
|
|
13
15
|
res.status(503).json({ status: "error", error: "YDB unavailable" });
|
|
14
16
|
scheduleExit(1);
|
|
15
17
|
return;
|
|
@@ -20,8 +22,8 @@ export async function healthHandler(_req, res) {
|
|
|
20
22
|
catch (err) {
|
|
21
23
|
const isTimeout = isCompilationTimeoutError(err);
|
|
22
24
|
logger.error({ err }, isTimeout
|
|
23
|
-
? "YDB compilation timeout during health probe
|
|
24
|
-
: "YDB health probe failed
|
|
25
|
+
? "YDB compilation timeout during health probe"
|
|
26
|
+
: "YDB health probe failed");
|
|
25
27
|
res.status(503).json({
|
|
26
28
|
status: "error",
|
|
27
29
|
error: "YDB health probe failed",
|
|
@@ -38,7 +40,15 @@ export function rootHandler(_req, res) {
|
|
|
38
40
|
export function buildServer() {
|
|
39
41
|
const app = express();
|
|
40
42
|
app.use(requestLogger);
|
|
41
|
-
app.use(
|
|
43
|
+
app.use(upsertBodyTimeout);
|
|
44
|
+
app.use(instrumentUpsertRequestBody);
|
|
45
|
+
app.use(express.json({
|
|
46
|
+
limit: "20mb",
|
|
47
|
+
verify: verifyUpsertRequestBody,
|
|
48
|
+
}));
|
|
49
|
+
app.use(logUpsertBodyPhaseError);
|
|
50
|
+
app.use(logUpsertBodyPhase);
|
|
51
|
+
app.use(upsertProcessingTimeout);
|
|
42
52
|
app.get("/", rootHandler);
|
|
43
53
|
app.get("/health", healthHandler);
|
|
44
54
|
app.use("/collections", collectionsRouter);
|
|
@@ -57,12 +67,16 @@ export function buildServer() {
|
|
|
57
67
|
});
|
|
58
68
|
// Catch-all error handler: avoid Express default handler printing stacktraces to stderr
|
|
59
69
|
// and provide consistent JSON error responses.
|
|
60
|
-
app.use((err,
|
|
61
|
-
logger.error({ err }, "Unhandled error in Express middleware");
|
|
70
|
+
app.use((err, req, res, _next) => {
|
|
62
71
|
void _next;
|
|
63
72
|
if (res.headersSent || res.writableEnded) {
|
|
73
|
+
if (shouldSuppressLateMiddlewareError(req, err)) {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
logger.error({ err }, "Late middleware error after response sent");
|
|
64
77
|
return;
|
|
65
78
|
}
|
|
79
|
+
logger.error({ err }, "Unhandled error in Express middleware");
|
|
66
80
|
const statusCode = extractHttpStatusCode(err) ?? 500;
|
|
67
81
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
68
82
|
res.status(statusCode).json({
|
|
@@ -85,6 +99,28 @@ function isRequestAbortedError(err) {
|
|
|
85
99
|
}
|
|
86
100
|
return false;
|
|
87
101
|
}
|
|
102
|
+
function isBodyParserRequestReadError(err) {
|
|
103
|
+
if (!err || typeof err !== "object") {
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
const typeValue = "type" in err && typeof err.type === "string" ? err.type : undefined;
|
|
107
|
+
if (!typeValue) {
|
|
108
|
+
return false;
|
|
109
|
+
}
|
|
110
|
+
return (typeValue === "entity.parse.failed" ||
|
|
111
|
+
typeValue === "entity.too.large" ||
|
|
112
|
+
typeValue === "request.size.invalid" ||
|
|
113
|
+
typeValue === "charset.unsupported" ||
|
|
114
|
+
typeValue === "encoding.unsupported" ||
|
|
115
|
+
typeValue === "stream.encoding.set" ||
|
|
116
|
+
typeValue === "stream.not.readable");
|
|
117
|
+
}
|
|
118
|
+
function shouldSuppressLateMiddlewareError(req, err) {
|
|
119
|
+
if (isRequestAbortedError(err)) {
|
|
120
|
+
return true;
|
|
121
|
+
}
|
|
122
|
+
return isUpsertRequestTimedOut(req) && isBodyParserRequestReadError(err);
|
|
123
|
+
}
|
|
88
124
|
function extractHttpStatusCode(err) {
|
|
89
125
|
if (!err || typeof err !== "object") {
|
|
90
126
|
return undefined;
|
|
@@ -1,23 +1,25 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { DistanceKind } from "../qdrant/QdrantRestTypes.js";
|
|
2
2
|
export interface CollectionContextInput {
|
|
3
|
-
|
|
3
|
+
userUid: string;
|
|
4
4
|
collection: string;
|
|
5
|
-
apiKey
|
|
5
|
+
apiKey: string;
|
|
6
6
|
userAgent?: string;
|
|
7
7
|
}
|
|
8
8
|
export interface NormalizedCollectionContext {
|
|
9
|
-
|
|
9
|
+
userUid: string;
|
|
10
10
|
collection: string;
|
|
11
11
|
metaKey: string;
|
|
12
|
+
uid: string;
|
|
12
13
|
}
|
|
13
14
|
export declare function putCollectionIndex(ctx: CollectionContextInput): Promise<{
|
|
14
15
|
acknowledged: boolean;
|
|
15
16
|
}>;
|
|
16
17
|
export declare function createCollection(ctx: CollectionContextInput, body: unknown): Promise<{
|
|
17
18
|
name: string;
|
|
18
|
-
tenant: string;
|
|
19
19
|
}>;
|
|
20
20
|
export declare function getCollection(ctx: CollectionContextInput): Promise<{
|
|
21
|
+
status: string;
|
|
22
|
+
points_count: number;
|
|
21
23
|
name: string;
|
|
22
24
|
vectors: {
|
|
23
25
|
size: number;
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { CreateCollectionReq } from "../
|
|
1
|
+
import { CreateCollectionReq } from "../qdrant/Requests.js";
|
|
2
2
|
import { ensureMetaTable } from "../ydb/schema.js";
|
|
3
|
-
import { createCollection as repoCreateCollection, deleteCollection as repoDeleteCollection, getCollectionMeta, touchCollectionLastAccess, } from "../repositories/collectionsRepo.js";
|
|
3
|
+
import { createCollection as repoCreateCollection, countPointsForCollection, deleteCollection as repoDeleteCollection, getCollectionMeta, touchCollectionLastAccess, } from "../repositories/collectionsRepo.js";
|
|
4
4
|
import { QdrantServiceError } from "./errors.js";
|
|
5
5
|
import { normalizeCollectionContextShared } from "./CollectionService.shared.js";
|
|
6
6
|
export async function putCollectionIndex(ctx) {
|
|
7
7
|
await ensureMetaTable();
|
|
8
|
-
const normalized = normalizeCollectionContextShared(ctx.
|
|
8
|
+
const normalized = normalizeCollectionContextShared(ctx.userUid, ctx.collection, ctx.userAgent);
|
|
9
9
|
const meta = await getCollectionMeta(normalized.metaKey);
|
|
10
10
|
if (!meta) {
|
|
11
11
|
throw new QdrantServiceError(404, {
|
|
@@ -18,7 +18,7 @@ export async function putCollectionIndex(ctx) {
|
|
|
18
18
|
}
|
|
19
19
|
export async function createCollection(ctx, body) {
|
|
20
20
|
await ensureMetaTable();
|
|
21
|
-
const normalized = normalizeCollectionContextShared(ctx.
|
|
21
|
+
const normalized = normalizeCollectionContextShared(ctx.userUid, ctx.collection, ctx.userAgent);
|
|
22
22
|
const parsed = CreateCollectionReq.safeParse(body);
|
|
23
23
|
if (!parsed.success) {
|
|
24
24
|
throw new QdrantServiceError(400, {
|
|
@@ -35,7 +35,7 @@ export async function createCollection(ctx, body) {
|
|
|
35
35
|
existing.distance === distance &&
|
|
36
36
|
existing.vectorType === vectorType) {
|
|
37
37
|
await touchCollectionLastAccess(normalized.metaKey);
|
|
38
|
-
return { name: normalized.collection
|
|
38
|
+
return { name: normalized.collection };
|
|
39
39
|
}
|
|
40
40
|
const errorMessage = `Collection already exists with different config: dimension=${existing.dimension}, distance=${existing.distance}, type=${existing.vectorType}`;
|
|
41
41
|
throw new QdrantServiceError(400, {
|
|
@@ -43,12 +43,12 @@ export async function createCollection(ctx, body) {
|
|
|
43
43
|
error: errorMessage,
|
|
44
44
|
});
|
|
45
45
|
}
|
|
46
|
-
await repoCreateCollection(normalized.metaKey, dim, distance, vectorType);
|
|
47
|
-
return { name: normalized.collection
|
|
46
|
+
await repoCreateCollection(normalized.metaKey, dim, distance, vectorType, ctx.userUid);
|
|
47
|
+
return { name: normalized.collection };
|
|
48
48
|
}
|
|
49
49
|
export async function getCollection(ctx) {
|
|
50
50
|
await ensureMetaTable();
|
|
51
|
-
const normalized = normalizeCollectionContextShared(ctx.
|
|
51
|
+
const normalized = normalizeCollectionContextShared(ctx.userUid, ctx.collection, ctx.userAgent);
|
|
52
52
|
const meta = await getCollectionMeta(normalized.metaKey);
|
|
53
53
|
if (!meta) {
|
|
54
54
|
throw new QdrantServiceError(404, {
|
|
@@ -56,8 +56,11 @@ export async function getCollection(ctx) {
|
|
|
56
56
|
error: "collection not found",
|
|
57
57
|
});
|
|
58
58
|
}
|
|
59
|
+
const pointsCount = await countPointsForCollection(normalized.uid);
|
|
59
60
|
await touchCollectionLastAccess(normalized.metaKey);
|
|
60
61
|
return {
|
|
62
|
+
status: "green",
|
|
63
|
+
points_count: pointsCount,
|
|
61
64
|
name: normalized.collection,
|
|
62
65
|
vectors: {
|
|
63
66
|
size: meta.dimension,
|
|
@@ -77,7 +80,7 @@ export async function getCollection(ctx) {
|
|
|
77
80
|
}
|
|
78
81
|
export async function deleteCollection(ctx) {
|
|
79
82
|
await ensureMetaTable();
|
|
80
|
-
const normalized = normalizeCollectionContextShared(ctx.
|
|
83
|
+
const normalized = normalizeCollectionContextShared(ctx.userUid, ctx.collection, ctx.userAgent);
|
|
81
84
|
await repoDeleteCollection(normalized.metaKey);
|
|
82
85
|
return { acknowledged: true };
|
|
83
86
|
}
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import { uidFor, } from "./CollectionService.shared.js";
|
|
2
1
|
import { GLOBAL_POINTS_TABLE, ensureGlobalPointsTable } from "../ydb/schema.js";
|
|
3
2
|
export async function resolvePointsTableAndUidOneTable(ctx) {
|
|
4
3
|
await ensureGlobalPointsTable();
|
|
5
4
|
return {
|
|
6
5
|
tableName: GLOBAL_POINTS_TABLE,
|
|
7
|
-
uid:
|
|
6
|
+
uid: ctx.uid,
|
|
8
7
|
};
|
|
9
8
|
}
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
export interface NormalizedCollectionContextLike {
|
|
2
|
-
|
|
2
|
+
userUid: string;
|
|
3
3
|
collection: string;
|
|
4
4
|
metaKey: string;
|
|
5
|
+
uid: string;
|
|
5
6
|
}
|
|
6
|
-
export declare function
|
|
7
|
-
export declare function
|
|
8
|
-
|
|
9
|
-
tenant: string;
|
|
7
|
+
export declare function uidForCollection(userUid: string, collection: string): string;
|
|
8
|
+
export declare function normalizeCollectionContextShared(userUid: string, collection: string, _userAgent?: string): {
|
|
9
|
+
userUid: string;
|
|
10
10
|
collection: string;
|
|
11
11
|
metaKey: string;
|
|
12
|
+
uid: string;
|
|
12
13
|
};
|
|
@@ -1,19 +1,35 @@
|
|
|
1
|
-
import { sanitizeCollectionName,
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
import { sanitizeCollectionName, metaKeyFor, uidFor, } from "../utils/tenant.js";
|
|
2
|
+
import { QdrantServiceError } from "./errors.js";
|
|
3
|
+
function requireAndSanitizeUserUid(userUid) {
|
|
4
|
+
if (!userUid || userUid.trim() === "") {
|
|
5
|
+
throw new QdrantServiceError(401, {
|
|
6
|
+
status: "error",
|
|
7
|
+
error: "unauthorized",
|
|
8
|
+
});
|
|
9
|
+
}
|
|
10
|
+
const raw = userUid.toString();
|
|
11
|
+
const cleaned = raw.replace(/[^a-zA-Z0-9_]/g, "_").replace(/_+/g, "_");
|
|
12
|
+
const lowered = cleaned.toLowerCase().replace(/^_+/, "");
|
|
13
|
+
if (lowered.length === 0) {
|
|
14
|
+
throw new QdrantServiceError(401, {
|
|
15
|
+
status: "error",
|
|
16
|
+
error: "unauthorized",
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
return lowered;
|
|
4
20
|
}
|
|
5
|
-
export function
|
|
6
|
-
return
|
|
21
|
+
export function uidForCollection(userUid, collection) {
|
|
22
|
+
return uidFor(userUid, collection);
|
|
7
23
|
}
|
|
8
|
-
export function normalizeCollectionContextShared(
|
|
9
|
-
|
|
10
|
-
const
|
|
11
|
-
const
|
|
12
|
-
const
|
|
13
|
-
const metaKey = metaKeyFor(normalizedTenant, normalizedCollection);
|
|
24
|
+
export function normalizeCollectionContextShared(userUid, collection, _userAgent) {
|
|
25
|
+
void _userAgent;
|
|
26
|
+
const normalizedUserUid = requireAndSanitizeUserUid(userUid);
|
|
27
|
+
const normalizedCollection = sanitizeCollectionName(collection);
|
|
28
|
+
const metaKey = metaKeyFor(normalizedUserUid, normalizedCollection);
|
|
14
29
|
return {
|
|
15
|
-
|
|
30
|
+
userUid: normalizedUserUid,
|
|
16
31
|
collection: normalizedCollection,
|
|
17
32
|
metaKey,
|
|
33
|
+
uid: uidForCollection(normalizedUserUid, normalizedCollection),
|
|
18
34
|
};
|
|
19
35
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type CollectionContextInput } from "./CollectionService.js";
|
|
2
2
|
import type { YdbQdrantScoredPoint } from "../qdrant/QdrantRestTypes.js";
|
|
3
|
+
import type { RetrievedPoint } from "../repositories/pointsRepo.one-table/Retrieve.js";
|
|
3
4
|
type PointsContextInput = CollectionContextInput;
|
|
4
5
|
export declare function upsertPoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
5
6
|
upserted: number;
|
|
@@ -10,6 +11,13 @@ export declare function searchPoints(ctx: PointsContextInput, body: unknown): Pr
|
|
|
10
11
|
export declare function queryPoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
11
12
|
points: YdbQdrantScoredPoint[];
|
|
12
13
|
}>;
|
|
14
|
+
export declare function retrievePoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
15
|
+
points: RetrievedPoint[];
|
|
16
|
+
}>;
|
|
17
|
+
/**
|
|
18
|
+
* @returns `deleted` — number of points removed, or `-1` when the exact count
|
|
19
|
+
* is unavailable (e.g. bulk BATCH DELETE or clear-all-points).
|
|
20
|
+
*/
|
|
13
21
|
export declare function deletePoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
14
22
|
deleted: number;
|
|
15
23
|
}>;
|