ydb-qdrant 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +222 -0
- package/dist/Api.d.ts +26 -0
- package/dist/Api.js +68 -0
- package/dist/SmokeTest.d.ts +1 -0
- package/dist/SmokeTest.js +40 -0
- package/dist/config/env.d.ts +6 -0
- package/dist/config/env.js +8 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +21 -0
- package/dist/indexing/IndexScheduler.d.ts +5 -0
- package/dist/indexing/IndexScheduler.js +66 -0
- package/dist/logging/logger.d.ts +2 -0
- package/dist/logging/logger.js +3 -0
- package/dist/middleware/requestLogger.d.ts +2 -0
- package/dist/middleware/requestLogger.js +14 -0
- package/dist/repositories/collectionsRepo.d.ts +10 -0
- package/dist/repositories/collectionsRepo.js +118 -0
- package/dist/repositories/pointsRepo.d.ts +12 -0
- package/dist/repositories/pointsRepo.js +151 -0
- package/dist/routes/collections.d.ts +1 -0
- package/dist/routes/collections.js +74 -0
- package/dist/routes/points.d.ts +1 -0
- package/dist/routes/points.js +96 -0
- package/dist/server.d.ts +1 -0
- package/dist/server.js +13 -0
- package/dist/services/QdrantService.d.ts +55 -0
- package/dist/services/QdrantService.js +268 -0
- package/dist/types.d.ts +28 -0
- package/dist/types.js +30 -0
- package/dist/utils/tenant.d.ts +4 -0
- package/dist/utils/tenant.js +17 -0
- package/dist/ydb/client.d.ts +6 -0
- package/dist/ydb/client.js +20 -0
- package/dist/ydb/helpers.d.ts +2 -0
- package/dist/ydb/helpers.js +45 -0
- package/dist/ydb/schema.d.ts +1 -0
- package/dist/ydb/schema.js +24 -0
- package/package.json +73 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { buildVectorIndex } from "../repositories/collectionsRepo.js";
|
|
2
|
+
import { logger } from "../logging/logger.js";
|
|
3
|
+
const QUIET_MS = 10000; // no upserts for 10s => build
|
|
4
|
+
const MIN_POINTS_THRESHOLD = 100; // only rebuild if at least this many points upserted
|
|
5
|
+
const state = {};
|
|
6
|
+
export function notifyUpsert(tableName, count = 1) {
|
|
7
|
+
const now = Date.now();
|
|
8
|
+
const s = state[tableName] ?? {
|
|
9
|
+
lastUpsertMs: 0,
|
|
10
|
+
pending: false,
|
|
11
|
+
pointsUpserted: 0,
|
|
12
|
+
};
|
|
13
|
+
s.lastUpsertMs = now;
|
|
14
|
+
s.pointsUpserted += count;
|
|
15
|
+
state[tableName] = s;
|
|
16
|
+
}
|
|
17
|
+
export function requestIndexBuild(tableName, dimension, distance, vectorType, opts) {
|
|
18
|
+
const s = state[tableName] ?? {
|
|
19
|
+
lastUpsertMs: 0,
|
|
20
|
+
pending: false,
|
|
21
|
+
pointsUpserted: 0,
|
|
22
|
+
};
|
|
23
|
+
state[tableName] = s;
|
|
24
|
+
if (opts?.force) {
|
|
25
|
+
logger.info({ tableName }, "index build (force) starting");
|
|
26
|
+
void buildVectorIndex(tableName, dimension, distance, vectorType)
|
|
27
|
+
.then(() => {
|
|
28
|
+
logger.info({ tableName }, "index build (force) completed");
|
|
29
|
+
s.pointsUpserted = 0;
|
|
30
|
+
})
|
|
31
|
+
.catch((err) => {
|
|
32
|
+
logger.error({ err, tableName }, "index build (force) failed");
|
|
33
|
+
});
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
if (s.pending && s.timer) {
|
|
37
|
+
// already scheduled; timer will check quiet window
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
s.pending = true;
|
|
41
|
+
s.timer = setTimeout(function tryBuild() {
|
|
42
|
+
const since = Date.now() - (state[tableName]?.lastUpsertMs ?? 0);
|
|
43
|
+
if (since < QUIET_MS) {
|
|
44
|
+
s.timer = setTimeout(tryBuild, QUIET_MS - since);
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const pointsCount = state[tableName]?.pointsUpserted ?? 0;
|
|
48
|
+
if (pointsCount < MIN_POINTS_THRESHOLD) {
|
|
49
|
+
logger.info({ tableName, pointsCount, threshold: MIN_POINTS_THRESHOLD }, "index build skipped (below threshold)");
|
|
50
|
+
s.pending = false;
|
|
51
|
+
s.timer = undefined;
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
logger.info({ tableName, pointsCount }, "index build (scheduled) starting");
|
|
55
|
+
void buildVectorIndex(tableName, dimension, distance, vectorType)
|
|
56
|
+
.then(() => {
|
|
57
|
+
logger.info({ tableName }, "index build (scheduled) completed");
|
|
58
|
+
state[tableName].pointsUpserted = 0;
|
|
59
|
+
})
|
|
60
|
+
.catch((err) => logger.error({ err, tableName }, "index build (scheduled) failed"))
|
|
61
|
+
.finally(() => {
|
|
62
|
+
s.pending = false;
|
|
63
|
+
s.timer = undefined;
|
|
64
|
+
});
|
|
65
|
+
}, QUIET_MS);
|
|
66
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { logger } from "../logging/logger.js";
|
|
2
|
+
export function requestLogger(req, res, next) {
|
|
3
|
+
const start = Date.now();
|
|
4
|
+
const tenant = req.header("X-Tenant-Id") ?? "default";
|
|
5
|
+
const { method, originalUrl } = req;
|
|
6
|
+
const contentLength = req.header("content-length");
|
|
7
|
+
const queryKeys = Object.keys(req.query || {});
|
|
8
|
+
logger.info({ method, url: originalUrl, tenant, contentLength, queryKeys }, "req");
|
|
9
|
+
res.on("finish", () => {
|
|
10
|
+
const ms = Date.now() - start;
|
|
11
|
+
logger.info({ method, url: originalUrl, tenant, status: res.statusCode, ms }, "res");
|
|
12
|
+
});
|
|
13
|
+
next();
|
|
14
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { DistanceKind, VectorType } from "../types";
|
|
2
|
+
export declare function createCollection(metaKey: string, dim: number, distance: DistanceKind, vectorType: VectorType, tableName: string): Promise<void>;
|
|
3
|
+
export declare function getCollectionMeta(metaKey: string): Promise<{
|
|
4
|
+
table: string;
|
|
5
|
+
dimension: number;
|
|
6
|
+
distance: DistanceKind;
|
|
7
|
+
vectorType: VectorType;
|
|
8
|
+
} | null>;
|
|
9
|
+
export declare function deleteCollection(metaKey: string): Promise<void>;
|
|
10
|
+
export declare function buildVectorIndex(tableName: string, dimension: number, distance: DistanceKind, vectorType: VectorType): Promise<void>;
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { Types, TypedValues, withSession, TableDescription, Column, } from "../ydb/client.js";
|
|
2
|
+
export async function createCollection(metaKey, dim, distance, vectorType, tableName) {
|
|
3
|
+
await withSession(async (s) => {
|
|
4
|
+
const desc = new TableDescription()
|
|
5
|
+
.withColumns(new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
|
|
6
|
+
.withPrimaryKey("point_id");
|
|
7
|
+
await s.createTable(tableName, desc);
|
|
8
|
+
});
|
|
9
|
+
const upsertMeta = `
|
|
10
|
+
DECLARE $collection AS Utf8;
|
|
11
|
+
DECLARE $table AS Utf8;
|
|
12
|
+
DECLARE $dim AS Uint32;
|
|
13
|
+
DECLARE $distance AS Utf8;
|
|
14
|
+
DECLARE $vtype AS Utf8;
|
|
15
|
+
DECLARE $created AS Timestamp;
|
|
16
|
+
UPSERT INTO qdr__collections (collection, table_name, vector_dimension, distance, vector_type, created_at)
|
|
17
|
+
VALUES ($collection, $table, $dim, $distance, $vtype, $created);
|
|
18
|
+
`;
|
|
19
|
+
await withSession(async (s) => {
|
|
20
|
+
await s.executeQuery(upsertMeta, {
|
|
21
|
+
$collection: TypedValues.utf8(metaKey),
|
|
22
|
+
$table: TypedValues.utf8(tableName),
|
|
23
|
+
$dim: TypedValues.uint32(dim),
|
|
24
|
+
$distance: TypedValues.utf8(distance),
|
|
25
|
+
$vtype: TypedValues.utf8(vectorType),
|
|
26
|
+
$created: TypedValues.timestamp(new Date()),
|
|
27
|
+
});
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
export async function getCollectionMeta(metaKey) {
|
|
31
|
+
const qry = `
|
|
32
|
+
DECLARE $collection AS Utf8;
|
|
33
|
+
SELECT table_name, vector_dimension, distance, vector_type
|
|
34
|
+
FROM qdr__collections
|
|
35
|
+
WHERE collection = $collection;
|
|
36
|
+
`;
|
|
37
|
+
const res = await withSession(async (s) => {
|
|
38
|
+
return await s.executeQuery(qry, {
|
|
39
|
+
$collection: TypedValues.utf8(metaKey),
|
|
40
|
+
});
|
|
41
|
+
});
|
|
42
|
+
const rowset = res.resultSets?.[0];
|
|
43
|
+
if (!rowset || rowset.rows?.length !== 1)
|
|
44
|
+
return null;
|
|
45
|
+
const row = rowset.rows[0];
|
|
46
|
+
const table = row.items?.[0]?.textValue;
|
|
47
|
+
const dimension = Number(row.items?.[1]?.uint32Value ?? row.items?.[1]?.textValue);
|
|
48
|
+
const distance = row.items?.[2]?.textValue ?? "Cosine";
|
|
49
|
+
const vectorType = row.items?.[3]?.textValue ?? "float";
|
|
50
|
+
return { table, dimension, distance, vectorType };
|
|
51
|
+
}
|
|
52
|
+
export async function deleteCollection(metaKey) {
|
|
53
|
+
const meta = await getCollectionMeta(metaKey);
|
|
54
|
+
if (!meta)
|
|
55
|
+
return;
|
|
56
|
+
await withSession(async (s) => {
|
|
57
|
+
await s.dropTable(meta.table);
|
|
58
|
+
});
|
|
59
|
+
const delMeta = `
|
|
60
|
+
DECLARE $collection AS Utf8;
|
|
61
|
+
DELETE FROM qdr__collections WHERE collection = $collection;
|
|
62
|
+
`;
|
|
63
|
+
await withSession(async (s) => {
|
|
64
|
+
await s.executeQuery(delMeta, { $collection: TypedValues.utf8(metaKey) });
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
export async function buildVectorIndex(tableName, dimension, distance, vectorType) {
|
|
68
|
+
const distParam = mapDistanceToIndexParam(distance);
|
|
69
|
+
// defaults for <100k vectors
|
|
70
|
+
const levels = 1;
|
|
71
|
+
const clusters = 128;
|
|
72
|
+
await withSession(async (s) => {
|
|
73
|
+
// Drop existing index if present
|
|
74
|
+
const dropDdl = `ALTER TABLE ${tableName} DROP INDEX emb_idx;`;
|
|
75
|
+
try {
|
|
76
|
+
const dropReq = { sessionId: s.sessionId, yqlText: dropDdl };
|
|
77
|
+
await s.api.executeSchemeQuery(dropReq);
|
|
78
|
+
}
|
|
79
|
+
catch (e) {
|
|
80
|
+
const msg = String(e?.message ?? e);
|
|
81
|
+
// ignore if index doesn't exist
|
|
82
|
+
if (!/not found|does not exist|no such index/i.test(msg)) {
|
|
83
|
+
throw e;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
// Create new index
|
|
87
|
+
const createDdl = `
|
|
88
|
+
ALTER TABLE ${tableName}
|
|
89
|
+
ADD INDEX emb_idx GLOBAL SYNC USING vector_kmeans_tree
|
|
90
|
+
ON (embedding)
|
|
91
|
+
WITH (
|
|
92
|
+
${distParam === "inner_product"
|
|
93
|
+
? `similarity="inner_product"`
|
|
94
|
+
: `distance="${distParam}"`},
|
|
95
|
+
vector_type="${vectorType}",
|
|
96
|
+
vector_dimension=${dimension},
|
|
97
|
+
clusters=${clusters},
|
|
98
|
+
levels=${levels}
|
|
99
|
+
);
|
|
100
|
+
`;
|
|
101
|
+
const createReq = { sessionId: s.sessionId, yqlText: createDdl };
|
|
102
|
+
await s.api.executeSchemeQuery(createReq);
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
function mapDistanceToIndexParam(distance) {
|
|
106
|
+
switch (distance) {
|
|
107
|
+
case "Cosine":
|
|
108
|
+
return "cosine";
|
|
109
|
+
case "Dot":
|
|
110
|
+
return "inner_product";
|
|
111
|
+
case "Euclid":
|
|
112
|
+
return "euclidean";
|
|
113
|
+
case "Manhattan":
|
|
114
|
+
return "manhattan";
|
|
115
|
+
default:
|
|
116
|
+
return "cosine";
|
|
117
|
+
}
|
|
118
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { VectorType, DistanceKind } from "../types";
|
|
2
|
+
export declare function upsertPoints(tableName: string, points: Array<{
|
|
3
|
+
id: string | number;
|
|
4
|
+
vector: number[];
|
|
5
|
+
payload?: Record<string, unknown>;
|
|
6
|
+
}>, vectorType: VectorType, dimension: number): Promise<number>;
|
|
7
|
+
export declare function searchPoints(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, vectorType: VectorType, dimension: number): Promise<Array<{
|
|
8
|
+
id: string;
|
|
9
|
+
score: number;
|
|
10
|
+
payload?: Record<string, unknown>;
|
|
11
|
+
}>>;
|
|
12
|
+
export declare function deletePoints(tableName: string, ids: Array<string | number>): Promise<number>;
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { TypedValues, withSession } from "../ydb/client.js";
|
|
2
|
+
import { buildJsonOrEmpty, buildVectorParam } from "../ydb/helpers.js";
|
|
3
|
+
import { logger } from "../logging/logger.js";
|
|
4
|
+
import { APPROX_PRESELECT } from "../config/env.js";
|
|
5
|
+
import { notifyUpsert } from "../indexing/IndexScheduler.js";
|
|
6
|
+
export async function upsertPoints(tableName, points, vectorType, dimension) {
|
|
7
|
+
let upserted = 0;
|
|
8
|
+
await withSession(async (s) => {
|
|
9
|
+
for (const p of points) {
|
|
10
|
+
const id = String(p.id);
|
|
11
|
+
if (p.vector.length !== dimension) {
|
|
12
|
+
throw new Error(`Vector dimension mismatch for id=${id}: got ${p.vector.length}, expected ${dimension}`);
|
|
13
|
+
}
|
|
14
|
+
const ddl = `
|
|
15
|
+
DECLARE $id AS Utf8;
|
|
16
|
+
DECLARE $vec AS List<${vectorType === "uint8" ? "Uint8" : "Float"}>;
|
|
17
|
+
DECLARE $payload AS JsonDocument;
|
|
18
|
+
UPSERT INTO ${tableName} (point_id, embedding, payload)
|
|
19
|
+
VALUES (
|
|
20
|
+
$id,
|
|
21
|
+
Untag(Knn::ToBinaryString${vectorType === "uint8" ? "Uint8" : "Float"}($vec), "${vectorType === "uint8" ? "Uint8Vector" : "FloatVector"}"),
|
|
22
|
+
$payload
|
|
23
|
+
);
|
|
24
|
+
`;
|
|
25
|
+
const params = {
|
|
26
|
+
$id: TypedValues.utf8(id),
|
|
27
|
+
$vec: buildVectorParam(p.vector, vectorType),
|
|
28
|
+
$payload: buildJsonOrEmpty(p.payload),
|
|
29
|
+
};
|
|
30
|
+
// Retry on transient schema/metadata mismatches during index rebuild
|
|
31
|
+
const maxRetries = 6; // ~ up to ~ (0.25 + jitter) * 2^5 ≈ few seconds
|
|
32
|
+
let attempt = 0;
|
|
33
|
+
// eslint-disable-next-line no-constant-condition
|
|
34
|
+
while (true) {
|
|
35
|
+
try {
|
|
36
|
+
await s.executeQuery(ddl, params);
|
|
37
|
+
break;
|
|
38
|
+
}
|
|
39
|
+
catch (e) {
|
|
40
|
+
const msg = String(e?.message ?? e);
|
|
41
|
+
const isTransient = /Aborted|schema version mismatch|Table metadata loading|Failed to load metadata/i.test(msg);
|
|
42
|
+
if (!isTransient || attempt >= maxRetries) {
|
|
43
|
+
throw e;
|
|
44
|
+
}
|
|
45
|
+
const backoffMs = Math.floor(250 * Math.pow(2, attempt) + Math.random() * 100);
|
|
46
|
+
logger.warn({ tableName, id, attempt, backoffMs }, "upsert aborted due to schema/metadata change; retrying");
|
|
47
|
+
await new Promise((r) => setTimeout(r, backoffMs));
|
|
48
|
+
attempt += 1;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
upserted += 1;
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
// notify scheduler for potential end-of-batch index build
|
|
55
|
+
notifyUpsert(tableName, upserted);
|
|
56
|
+
// No index rebuild; approximate search does not require it
|
|
57
|
+
return upserted;
|
|
58
|
+
}
|
|
59
|
+
// Removed legacy index backfill helper
|
|
60
|
+
export async function searchPoints(tableName, queryVector, top, withPayload, distance, vectorType, dimension) {
|
|
61
|
+
if (queryVector.length !== dimension) {
|
|
62
|
+
throw new Error(`Vector dimension mismatch: got ${queryVector.length}, expected ${dimension}`);
|
|
63
|
+
}
|
|
64
|
+
const { fn, order } = mapDistanceToKnnFn(distance);
|
|
65
|
+
// Single-phase search over embedding using vector index if present
|
|
66
|
+
const preselect = Math.min(APPROX_PRESELECT, Math.max(top * 10, top));
|
|
67
|
+
const qf = buildVectorParam(queryVector, vectorType);
|
|
68
|
+
const params = {
|
|
69
|
+
$qf: qf,
|
|
70
|
+
$k2: TypedValues.uint32(top),
|
|
71
|
+
};
|
|
72
|
+
const buildQuery = (useIndex) => `
|
|
73
|
+
DECLARE $qf AS List<${vectorType === "uint8" ? "Uint8" : "Float"}>;
|
|
74
|
+
DECLARE $k2 AS Uint32;
|
|
75
|
+
$qbinf = Knn::ToBinaryString${vectorType === "uint8" ? "Uint8" : "Float"}($qf);
|
|
76
|
+
SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
|
|
77
|
+
FROM ${tableName}${useIndex ? " VIEW emb_idx" : ""}
|
|
78
|
+
ORDER BY score ${order}
|
|
79
|
+
LIMIT $k2;
|
|
80
|
+
`;
|
|
81
|
+
let rs;
|
|
82
|
+
try {
|
|
83
|
+
// Try with vector index first
|
|
84
|
+
rs = await withSession(async (s) => {
|
|
85
|
+
return await s.executeQuery(buildQuery(true), params);
|
|
86
|
+
});
|
|
87
|
+
logger.info({ tableName }, "vector index found; using index for search");
|
|
88
|
+
}
|
|
89
|
+
catch (e) {
|
|
90
|
+
const msg = String(e?.message ?? e);
|
|
91
|
+
// Fallback to table scan if index not found or not ready
|
|
92
|
+
if (/not found|does not exist|no such index|no global index|is not ready to use/i.test(msg)) {
|
|
93
|
+
logger.info({ tableName }, "vector index not available (missing or building); falling back to table scan");
|
|
94
|
+
rs = await withSession(async (s) => {
|
|
95
|
+
return await s.executeQuery(buildQuery(false), params);
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
throw e;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const rowset = rs.resultSets?.[0];
|
|
103
|
+
const rows = (rowset?.rows ?? []);
|
|
104
|
+
return rows.map((row) => {
|
|
105
|
+
const id = row.items?.[0]?.textValue;
|
|
106
|
+
let payload;
|
|
107
|
+
let scoreIdx = 1;
|
|
108
|
+
if (withPayload) {
|
|
109
|
+
const payloadText = row.items?.[1]?.textValue;
|
|
110
|
+
if (payloadText) {
|
|
111
|
+
try {
|
|
112
|
+
payload = JSON.parse(payloadText);
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
payload = undefined;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
scoreIdx = 2;
|
|
119
|
+
}
|
|
120
|
+
const score = Number(row.items?.[scoreIdx]?.floatValue ?? row.items?.[scoreIdx]?.textValue);
|
|
121
|
+
return { id, score, ...(payload ? { payload } : {}) };
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
export async function deletePoints(tableName, ids) {
|
|
125
|
+
let deleted = 0;
|
|
126
|
+
await withSession(async (s) => {
|
|
127
|
+
for (const id of ids) {
|
|
128
|
+
const yql = `
|
|
129
|
+
DECLARE $id AS Utf8;
|
|
130
|
+
DELETE FROM ${tableName} WHERE point_id = $id;
|
|
131
|
+
`;
|
|
132
|
+
await s.executeQuery(yql, { $id: TypedValues.utf8(String(id)) });
|
|
133
|
+
deleted += 1;
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
return deleted;
|
|
137
|
+
}
|
|
138
|
+
function mapDistanceToKnnFn(distance) {
|
|
139
|
+
switch (distance) {
|
|
140
|
+
case "Cosine":
|
|
141
|
+
return { fn: "Knn::CosineSimilarity", order: "DESC" };
|
|
142
|
+
case "Dot":
|
|
143
|
+
return { fn: "Knn::InnerProductSimilarity", order: "DESC" };
|
|
144
|
+
case "Euclid":
|
|
145
|
+
return { fn: "Knn::EuclideanDistance", order: "ASC" };
|
|
146
|
+
case "Manhattan":
|
|
147
|
+
return { fn: "Knn::ManhattanDistance", order: "ASC" };
|
|
148
|
+
default:
|
|
149
|
+
return { fn: "Knn::CosineSimilarity", order: "DESC" };
|
|
150
|
+
}
|
|
151
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const collectionsRouter: import("express-serve-static-core").Router;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { Router } from "express";
|
|
2
|
+
import { sanitizeCollectionName, sanitizeTenantId } from "../utils/tenant.js";
|
|
3
|
+
import { QdrantServiceError, putCollectionIndex, createCollection, getCollection, deleteCollection, } from "../services/QdrantService.js";
|
|
4
|
+
import { logger } from "../logging/logger.js";
|
|
5
|
+
export const collectionsRouter = Router();
|
|
6
|
+
collectionsRouter.put("/:collection/index", async (req, res) => {
|
|
7
|
+
try {
|
|
8
|
+
const result = await putCollectionIndex({
|
|
9
|
+
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
10
|
+
collection: String(req.params.collection),
|
|
11
|
+
});
|
|
12
|
+
res.json({ status: "ok", result });
|
|
13
|
+
}
|
|
14
|
+
catch (err) {
|
|
15
|
+
if (err instanceof QdrantServiceError) {
|
|
16
|
+
return res.status(err.statusCode).json(err.payload);
|
|
17
|
+
}
|
|
18
|
+
logger.error({ err }, "build index failed");
|
|
19
|
+
res
|
|
20
|
+
.status(500)
|
|
21
|
+
.json({ status: "error", error: String(err?.message ?? err) });
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
collectionsRouter.put("/:collection", async (req, res) => {
|
|
25
|
+
try {
|
|
26
|
+
const tenant = sanitizeTenantId(req.header("X-Tenant-Id") ?? "default");
|
|
27
|
+
const collection = sanitizeCollectionName(String(req.params.collection));
|
|
28
|
+
const result = await createCollection({ tenant, collection }, req.body);
|
|
29
|
+
res.json({ status: "ok", result });
|
|
30
|
+
}
|
|
31
|
+
catch (err) {
|
|
32
|
+
if (err instanceof QdrantServiceError) {
|
|
33
|
+
return res.status(err.statusCode).json(err.payload);
|
|
34
|
+
}
|
|
35
|
+
logger.error({ err }, "create collection failed");
|
|
36
|
+
res
|
|
37
|
+
.status(500)
|
|
38
|
+
.json({ status: "error", error: String(err?.message ?? err) });
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
collectionsRouter.get("/:collection", async (req, res) => {
|
|
42
|
+
try {
|
|
43
|
+
const tenant = sanitizeTenantId(req.header("X-Tenant-Id") ?? "default");
|
|
44
|
+
const collection = sanitizeCollectionName(String(req.params.collection));
|
|
45
|
+
const result = await getCollection({ tenant, collection });
|
|
46
|
+
res.json({ status: "ok", result });
|
|
47
|
+
}
|
|
48
|
+
catch (err) {
|
|
49
|
+
if (err instanceof QdrantServiceError) {
|
|
50
|
+
return res.status(err.statusCode).json(err.payload);
|
|
51
|
+
}
|
|
52
|
+
logger.error({ err }, "get collection failed");
|
|
53
|
+
res
|
|
54
|
+
.status(500)
|
|
55
|
+
.json({ status: "error", error: String(err?.message ?? err) });
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
collectionsRouter.delete("/:collection", async (req, res) => {
|
|
59
|
+
try {
|
|
60
|
+
const tenant = sanitizeTenantId(req.header("X-Tenant-Id") ?? "default");
|
|
61
|
+
const collection = sanitizeCollectionName(String(req.params.collection));
|
|
62
|
+
const result = await deleteCollection({ tenant, collection });
|
|
63
|
+
res.json({ status: "ok", result });
|
|
64
|
+
}
|
|
65
|
+
catch (err) {
|
|
66
|
+
if (err instanceof QdrantServiceError) {
|
|
67
|
+
return res.status(err.statusCode).json(err.payload);
|
|
68
|
+
}
|
|
69
|
+
logger.error({ err }, "delete collection failed");
|
|
70
|
+
res
|
|
71
|
+
.status(500)
|
|
72
|
+
.json({ status: "error", error: String(err?.message ?? err) });
|
|
73
|
+
}
|
|
74
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const pointsRouter: import("express-serve-static-core").Router;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { Router } from "express";
|
|
2
|
+
import { QdrantServiceError, upsertPoints, searchPoints, queryPoints, deletePoints, } from "../services/QdrantService.js";
|
|
3
|
+
import { logger } from "../logging/logger.js";
|
|
4
|
+
export const pointsRouter = Router();
|
|
5
|
+
// Qdrant-compatible: PUT /collections/:collection/points (upsert)
|
|
6
|
+
pointsRouter.put("/:collection/points", async (req, res) => {
|
|
7
|
+
try {
|
|
8
|
+
const result = await upsertPoints({
|
|
9
|
+
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
10
|
+
collection: String(req.params.collection),
|
|
11
|
+
}, req.body);
|
|
12
|
+
res.json({ status: "ok", result });
|
|
13
|
+
}
|
|
14
|
+
catch (err) {
|
|
15
|
+
if (err instanceof QdrantServiceError) {
|
|
16
|
+
return res.status(err.statusCode).json(err.payload);
|
|
17
|
+
}
|
|
18
|
+
logger.error({ err }, "upsert points (PUT) failed");
|
|
19
|
+
res
|
|
20
|
+
.status(500)
|
|
21
|
+
.json({ status: "error", error: String(err?.message ?? err) });
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
pointsRouter.post("/:collection/points/upsert", async (req, res) => {
|
|
25
|
+
try {
|
|
26
|
+
const result = await upsertPoints({
|
|
27
|
+
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
28
|
+
collection: String(req.params.collection),
|
|
29
|
+
}, req.body);
|
|
30
|
+
res.json({ status: "ok", result });
|
|
31
|
+
}
|
|
32
|
+
catch (err) {
|
|
33
|
+
if (err instanceof QdrantServiceError) {
|
|
34
|
+
return res.status(err.statusCode).json(err.payload);
|
|
35
|
+
}
|
|
36
|
+
logger.error({ err }, "upsert points failed");
|
|
37
|
+
res
|
|
38
|
+
.status(500)
|
|
39
|
+
.json({ status: "error", error: String(err?.message ?? err) });
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
pointsRouter.post("/:collection/points/search", async (req, res) => {
|
|
43
|
+
try {
|
|
44
|
+
const result = await searchPoints({
|
|
45
|
+
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
46
|
+
collection: String(req.params.collection),
|
|
47
|
+
}, req.body);
|
|
48
|
+
res.json({ status: "ok", result });
|
|
49
|
+
}
|
|
50
|
+
catch (err) {
|
|
51
|
+
if (err instanceof QdrantServiceError) {
|
|
52
|
+
return res.status(err.statusCode).json(err.payload);
|
|
53
|
+
}
|
|
54
|
+
logger.error({ err }, "search points failed");
|
|
55
|
+
res
|
|
56
|
+
.status(500)
|
|
57
|
+
.json({ status: "error", error: String(err?.message ?? err) });
|
|
58
|
+
}
|
|
59
|
+
});
|
|
60
|
+
// Compatibility: some clients call POST /collections/:collection/points/query
|
|
61
|
+
pointsRouter.post("/:collection/points/query", async (req, res) => {
|
|
62
|
+
try {
|
|
63
|
+
const result = await queryPoints({
|
|
64
|
+
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
65
|
+
collection: String(req.params.collection),
|
|
66
|
+
}, req.body);
|
|
67
|
+
res.json({ status: "ok", result });
|
|
68
|
+
}
|
|
69
|
+
catch (err) {
|
|
70
|
+
if (err instanceof QdrantServiceError) {
|
|
71
|
+
return res.status(err.statusCode).json(err.payload);
|
|
72
|
+
}
|
|
73
|
+
logger.error({ err }, "search points (query) failed");
|
|
74
|
+
res
|
|
75
|
+
.status(500)
|
|
76
|
+
.json({ status: "error", error: String(err?.message ?? err) });
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
pointsRouter.post("/:collection/points/delete", async (req, res) => {
|
|
80
|
+
try {
|
|
81
|
+
const result = await deletePoints({
|
|
82
|
+
tenant: req.header("X-Tenant-Id") ?? undefined,
|
|
83
|
+
collection: String(req.params.collection),
|
|
84
|
+
}, req.body);
|
|
85
|
+
res.json({ status: "ok", result });
|
|
86
|
+
}
|
|
87
|
+
catch (err) {
|
|
88
|
+
if (err instanceof QdrantServiceError) {
|
|
89
|
+
return res.status(err.statusCode).json(err.payload);
|
|
90
|
+
}
|
|
91
|
+
logger.error({ err }, "delete points failed");
|
|
92
|
+
res
|
|
93
|
+
.status(500)
|
|
94
|
+
.json({ status: "error", error: String(err?.message ?? err) });
|
|
95
|
+
}
|
|
96
|
+
});
|
package/dist/server.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function buildServer(): import("express-serve-static-core").Express;
|
package/dist/server.js
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import express from "express";
|
|
2
|
+
import { collectionsRouter } from "./routes/collections.js";
|
|
3
|
+
import { pointsRouter } from "./routes/points.js";
|
|
4
|
+
import { requestLogger } from "./middleware/requestLogger.js";
|
|
5
|
+
export function buildServer() {
|
|
6
|
+
const app = express();
|
|
7
|
+
app.use(express.json({ limit: "20mb" }));
|
|
8
|
+
app.use(requestLogger);
|
|
9
|
+
app.get("/health", (_req, res) => res.json({ status: "ok" }));
|
|
10
|
+
app.use("/collections", collectionsRouter);
|
|
11
|
+
app.use("/collections", pointsRouter);
|
|
12
|
+
return app;
|
|
13
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { type DistanceKind } from "../types.js";
|
|
2
|
+
export interface QdrantServiceErrorPayload {
|
|
3
|
+
status: "error";
|
|
4
|
+
error: unknown;
|
|
5
|
+
}
|
|
6
|
+
export declare class QdrantServiceError extends Error {
|
|
7
|
+
readonly statusCode: number;
|
|
8
|
+
readonly payload: QdrantServiceErrorPayload;
|
|
9
|
+
constructor(statusCode: number, payload: QdrantServiceErrorPayload, message?: string);
|
|
10
|
+
}
|
|
11
|
+
interface CollectionContextInput {
|
|
12
|
+
tenant: string | undefined;
|
|
13
|
+
collection: string;
|
|
14
|
+
}
|
|
15
|
+
export declare function putCollectionIndex(ctx: CollectionContextInput): Promise<{
|
|
16
|
+
acknowledged: boolean;
|
|
17
|
+
}>;
|
|
18
|
+
export declare function createCollection(ctx: CollectionContextInput, body: unknown): Promise<{
|
|
19
|
+
name: string;
|
|
20
|
+
tenant: string;
|
|
21
|
+
}>;
|
|
22
|
+
export declare function getCollection(ctx: CollectionContextInput): Promise<{
|
|
23
|
+
name: string;
|
|
24
|
+
vectors: {
|
|
25
|
+
size: number;
|
|
26
|
+
distance: DistanceKind;
|
|
27
|
+
data_type: string;
|
|
28
|
+
};
|
|
29
|
+
}>;
|
|
30
|
+
export declare function deleteCollection(ctx: CollectionContextInput): Promise<{
|
|
31
|
+
acknowledged: boolean;
|
|
32
|
+
}>;
|
|
33
|
+
interface PointsContextInput extends CollectionContextInput {
|
|
34
|
+
}
|
|
35
|
+
export declare function upsertPoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
36
|
+
upserted: number;
|
|
37
|
+
}>;
|
|
38
|
+
export declare function searchPoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
39
|
+
points: Array<{
|
|
40
|
+
id: string;
|
|
41
|
+
score: number;
|
|
42
|
+
payload?: Record<string, unknown>;
|
|
43
|
+
}>;
|
|
44
|
+
}>;
|
|
45
|
+
export declare function queryPoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
46
|
+
points: Array<{
|
|
47
|
+
id: string;
|
|
48
|
+
score: number;
|
|
49
|
+
payload?: Record<string, unknown>;
|
|
50
|
+
}>;
|
|
51
|
+
}>;
|
|
52
|
+
export declare function deletePoints(ctx: PointsContextInput, body: unknown): Promise<{
|
|
53
|
+
deleted: number;
|
|
54
|
+
}>;
|
|
55
|
+
export {};
|