ydb-qdrant 6.0.0 → 8.1.0

package/dist/repositories/pointsRepo.one-table.js

@@ -1,403 +1,3 @@
-import { TypedValues, Types, withSession, createExecuteQuerySettings, createExecuteQuerySettingsWithTimeout, } from "../ydb/client.js";
-import { buildVectorParam, buildVectorBinaryParams } from "../ydb/helpers.js";
-import { mapDistanceToKnnFn, mapDistanceToBitKnnFn, } from "../utils/distance.js";
-import { withRetry, isTransientYdbError } from "../utils/retry.js";
-import { UPSERT_BATCH_SIZE } from "../ydb/schema.js";
-import { CLIENT_SIDE_SERIALIZATION_ENABLED, SearchMode, UPSERT_OPERATION_TIMEOUT_MS, SEARCH_OPERATION_TIMEOUT_MS, } from "../config/env.js";
-import { logger } from "../logging/logger.js";
-export async function upsertPointsOneTable(tableName, points, dimension, uid) {
-    for (const p of points) {
-        const id = String(p.id);
-        if (p.vector.length !== dimension) {
-            const previewLength = Math.min(16, p.vector.length);
-            const vectorPreview = previewLength > 0 ? p.vector.slice(0, previewLength) : [];
-            logger.warn({
-                tableName,
-                uid,
-                pointId: id,
-                vectorLen: p.vector.length,
-                expectedDimension: dimension,
-                vectorPreview,
-            }, "upsertPointsOneTable: vector dimension mismatch");
-            throw new Error(`Vector dimension mismatch for id=${id}: got ${p.vector.length}, expected ${dimension}`);
-        }
-    }
-    let upserted = 0;
-    await withSession(async (s) => {
-        const settings = createExecuteQuerySettingsWithTimeout({
-            keepInCache: true,
-            idempotent: true,
-            timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
-        });
-        for (let i = 0; i < points.length; i += UPSERT_BATCH_SIZE) {
-            const batch = points.slice(i, i + UPSERT_BATCH_SIZE);
-            let ddl;
-            let params;
-            if (CLIENT_SIDE_SERIALIZATION_ENABLED) {
-                ddl = `
-          DECLARE $rows AS List<Struct<
-            uid: Utf8,
-            point_id: Utf8,
-            embedding: String,
-            embedding_quantized: String,
-            payload: JsonDocument
-          >>;
-
-          UPSERT INTO ${tableName} (uid, point_id, embedding, embedding_quantized, payload)
-          SELECT
-            uid,
-            point_id,
-            embedding,
-            embedding_quantized,
-            payload
-          FROM AS_TABLE($rows);
-        `;
-                const rowType = Types.struct({
-                    uid: Types.UTF8,
-                    point_id: Types.UTF8,
-                    embedding: Types.BYTES,
-                    embedding_quantized: Types.BYTES,
-                    payload: Types.JSON_DOCUMENT,
-                });
-                const rowsValue = TypedValues.list(rowType, batch.map((p) => {
-                    const binaries = buildVectorBinaryParams(p.vector);
-                    return {
-                        uid,
-                        point_id: String(p.id),
-                        embedding: binaries.float,
-                        embedding_quantized: binaries.bit,
-                        payload: JSON.stringify(p.payload ?? {}),
-                    };
-                }));
-                params = {
-                    $rows: rowsValue,
-                };
-            }
-            else {
-                ddl = `
-          DECLARE $rows AS List<Struct<
-            uid: Utf8,
-            point_id: Utf8,
-            vec: List<Float>,
-            payload: JsonDocument
-          >>;
-
-          UPSERT INTO ${tableName} (uid, point_id, embedding, embedding_quantized, payload)
-          SELECT
-            uid,
-            point_id,
-            Untag(Knn::ToBinaryStringFloat(vec), "FloatVector") AS embedding,
-            Untag(Knn::ToBinaryStringBit(vec), "BitVector") AS embedding_quantized,
-            payload
-          FROM AS_TABLE($rows);
-        `;
-                const rowType = Types.struct({
-                    uid: Types.UTF8,
-                    point_id: Types.UTF8,
-                    vec: Types.list(Types.FLOAT),
-                    payload: Types.JSON_DOCUMENT,
-                });
-                const rowsValue = TypedValues.list(rowType, batch.map((p) => ({
-                    uid,
-                    point_id: String(p.id),
-                    vec: p.vector,
-                    payload: JSON.stringify(p.payload ?? {}),
-                })));
-                params = {
-                    $rows: rowsValue,
-                };
-            }
-            logger.debug({
-                tableName,
-                mode: CLIENT_SIDE_SERIALIZATION_ENABLED
-                    ? "one_table_upsert_client_side_serialization"
-                    : "one_table_upsert_server_side_knn",
-                batchSize: batch.length,
-                yql: ddl,
-                params: {
-                    rows: batch.map((p) => ({
-                        uid,
-                        point_id: String(p.id),
-                        vectorLength: p.vector.length,
-                        vectorPreview: p.vector.slice(0, 3),
-                        payload: p.payload ?? {},
-                    })),
-                },
-            }, "one_table upsert: executing YQL");
-            await withRetry(() => s.executeQuery(ddl, params, undefined, settings), {
-                isTransient: isTransientYdbError,
-                context: { tableName, batchSize: batch.length },
-            });
-            upserted += batch.length;
-        }
-    });
-    return upserted;
-}
-async function searchPointsOneTableExact(tableName, queryVector, top, withPayload, distance, dimension, uid) {
-    if (queryVector.length !== dimension) {
-        throw new Error(`Vector dimension mismatch: got ${queryVector.length}, expected ${dimension}`);
-    }
-    const { fn, order } = mapDistanceToKnnFn(distance);
-    const results = await withSession(async (s) => {
-        let yql;
-        let params;
-        if (CLIENT_SIDE_SERIALIZATION_ENABLED) {
-            const binaries = buildVectorBinaryParams(queryVector);
-            yql = `
-        DECLARE $qbinf AS String;
-        DECLARE $k AS Uint32;
-        DECLARE $uid AS Utf8;
-        SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
-        FROM ${tableName}
-        WHERE uid = $uid
-        ORDER BY score ${order}
-        LIMIT $k;
-      `;
-            params = {
-                $qbinf: typeof TypedValues.bytes === "function"
-                    ? TypedValues.bytes(binaries.float)
-                    : binaries.float,
-                $k: TypedValues.uint32(top),
-                $uid: TypedValues.utf8(uid),
-            };
-        }
-        else {
-            const qf = buildVectorParam(queryVector);
-            yql = `
-        DECLARE $qf AS List<Float>;
-        DECLARE $k AS Uint32;
-        DECLARE $uid AS Utf8;
-        $qbinf = Knn::ToBinaryStringFloat($qf);
-        SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
-        FROM ${tableName}
-        WHERE uid = $uid
-        ORDER BY score ${order}
-        LIMIT $k;
-      `;
-            params = {
-                $qf: qf,
-                $k: TypedValues.uint32(top),
-                $uid: TypedValues.utf8(uid),
-            };
-        }
-        logger.debug({
-            tableName,
-            distance,
-            top,
-            withPayload,
-            mode: CLIENT_SIDE_SERIALIZATION_ENABLED
-                ? "one_table_exact_client_side_serialization"
-                : "one_table_exact",
-            yql,
-            params: {
-                uid,
-                top,
-                vectorLength: queryVector.length,
-                vectorPreview: queryVector.slice(0, 3),
-            },
-        }, "one_table search (exact): executing YQL");
-        const settings = createExecuteQuerySettingsWithTimeout({
-            keepInCache: true,
-            idempotent: true,
-            timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
-        });
-        const rs = await s.executeQuery(yql, params, undefined, settings);
-        const rowset = rs.resultSets?.[0];
-        const rows = (rowset?.rows ?? []);
-        return rows.map((row) => {
-            const id = row.items?.[0]?.textValue;
-            if (typeof id !== "string") {
-                throw new Error("point_id is missing in YDB search result");
-            }
-            let payload;
-            let scoreIdx = 1;
-            if (withPayload) {
-                const payloadText = row.items?.[1]?.textValue;
-                if (payloadText) {
-                    try {
-                        payload = JSON.parse(payloadText);
-                    }
-                    catch {
-                        payload = undefined;
-                    }
-                }
-                scoreIdx = 2;
-            }
-            const score = Number(row.items?.[scoreIdx]?.floatValue ?? row.items?.[scoreIdx]?.textValue);
-            return { id, score, ...(payload ? { payload } : {}) };
-        });
-    });
-    return results;
-}
-async function searchPointsOneTableApproximate(tableName, queryVector, top, withPayload, distance, dimension, uid, overfetchMultiplier) {
-    if (queryVector.length !== dimension) {
-        throw new Error(`Vector dimension mismatch: got ${queryVector.length}, expected ${dimension}`);
-    }
-    const { fn, order } = mapDistanceToKnnFn(distance);
-    const { fn: bitFn, order: bitOrder } = mapDistanceToBitKnnFn(distance);
-    const safeTop = top > 0 ? top : 1;
-    const rawCandidateLimit = safeTop * overfetchMultiplier;
-    const candidateLimit = Math.max(safeTop, rawCandidateLimit);
-    const results = await withSession(async (s) => {
-        let yql;
-        let params;
-        if (CLIENT_SIDE_SERIALIZATION_ENABLED) {
-            const binaries = buildVectorBinaryParams(queryVector);
-            yql = `
-        DECLARE $qbin_bit AS String;
-        DECLARE $qbinf AS String;
-        DECLARE $candidateLimit AS Uint32;
-        DECLARE $safeTop AS Uint32;
-        DECLARE $uid AS Utf8;
-
-        $candidates = (
-          SELECT point_id
-          FROM ${tableName}
-          WHERE uid = $uid AND embedding_quantized IS NOT NULL
-          ORDER BY ${bitFn}(embedding_quantized, $qbin_bit) ${bitOrder}
-          LIMIT $candidateLimit
-        );
-
-        SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
-        FROM ${tableName}
-        WHERE uid = $uid
-          AND point_id IN $candidates
-        ORDER BY score ${order}
-        LIMIT $safeTop;
-      `;
-            params = {
-                $qbin_bit: typeof TypedValues.bytes === "function"
-                    ? TypedValues.bytes(binaries.bit)
-                    : binaries.bit,
-                $qbinf: typeof TypedValues.bytes === "function"
-                    ? TypedValues.bytes(binaries.float)
-                    : binaries.float,
-                $candidateLimit: TypedValues.uint32(candidateLimit),
-                $safeTop: TypedValues.uint32(safeTop),
-                $uid: TypedValues.utf8(uid),
-            };
-            logger.debug({
-                tableName,
-                distance,
-                top,
-                safeTop,
-                candidateLimit,
-                mode: "one_table_approximate_client_side_serialization",
-                yql,
-                params: {
-                    uid,
-                    safeTop,
-                    candidateLimit,
-                    vectorLength: queryVector.length,
-                    vectorPreview: queryVector.slice(0, 3),
-                },
-            }, "one_table search (approximate): executing YQL with client-side serialization");
-        }
-        else {
-            const qf = buildVectorParam(queryVector);
-            yql = `
-        DECLARE $qf AS List<Float>;
-        DECLARE $candidateLimit AS Uint32;
-        DECLARE $safeTop AS Uint32;
-        DECLARE $uid AS Utf8;
-
-        $qbin_bit = Knn::ToBinaryStringBit($qf);
-        $qbinf = Knn::ToBinaryStringFloat($qf);
-
-        $candidates = (
-          SELECT point_id
-          FROM ${tableName}
-          WHERE uid = $uid AND embedding_quantized IS NOT NULL
-          ORDER BY ${bitFn}(embedding_quantized, $qbin_bit) ${bitOrder}
-          LIMIT $candidateLimit
-        );
-
-        SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
-        FROM ${tableName}
-        WHERE uid = $uid
-          AND point_id IN $candidates
-        ORDER BY score ${order}
-        LIMIT $safeTop;
-      `;
-            params = {
-                $qf: qf,
-                $candidateLimit: TypedValues.uint32(candidateLimit),
-                $safeTop: TypedValues.uint32(safeTop),
-                $uid: TypedValues.utf8(uid),
-            };
-            logger.debug({
-                tableName,
-                distance,
-                top,
-                safeTop,
-                candidateLimit,
-                mode: "one_table_approximate",
-                yql,
-                params: {
-                    uid,
-                    safeTop,
-                    candidateLimit,
-                    vectorLength: queryVector.length,
-                    vectorPreview: queryVector.slice(0, 3),
-                },
-            }, "one_table search (approximate): executing YQL");
-        }
-        const settings = createExecuteQuerySettingsWithTimeout({
-            keepInCache: true,
-            idempotent: true,
-            timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
-        });
-        const rs = await s.executeQuery(yql, params, undefined, settings);
-        const rowset = rs.resultSets?.[0];
-        const rows = (rowset?.rows ?? []);
-        return rows.map((row) => {
-            const id = row.items?.[0]?.textValue;
-            if (typeof id !== "string") {
-                throw new Error("point_id is missing in YDB search result");
-            }
-            let payload;
-            let scoreIdx = 1;
-            if (withPayload) {
-                const payloadText = row.items?.[1]?.textValue;
-                if (payloadText) {
-                    try {
-                        payload = JSON.parse(payloadText);
-                    }
-                    catch {
-                        payload = undefined;
-                    }
-                }
-                scoreIdx = 2;
-            }
-            const score = Number(row.items?.[scoreIdx]?.floatValue ?? row.items?.[scoreIdx]?.textValue);
-            return { id, score, ...(payload ? { payload } : {}) };
-        });
-    });
-    return results;
-}
-export async function searchPointsOneTable(tableName, queryVector, top, withPayload, distance, dimension, uid, mode, overfetchMultiplier) {
-    if (mode === SearchMode.Exact) {
-        return await searchPointsOneTableExact(tableName, queryVector, top, withPayload, distance, dimension, uid);
-    }
-    return await searchPointsOneTableApproximate(tableName, queryVector, top, withPayload, distance, dimension, uid, overfetchMultiplier);
-}
-export async function deletePointsOneTable(tableName, ids, uid) {
-    let deleted = 0;
-    await withSession(async (s) => {
-        const settings = createExecuteQuerySettings();
-        for (const id of ids) {
-            const yql = `
-        DECLARE $uid AS Utf8;
-        DECLARE $id AS Utf8;
-        DELETE FROM ${tableName} WHERE uid = $uid AND point_id = $id;
-      `;
-            const params = {
-                $uid: TypedValues.utf8(uid),
-                $id: TypedValues.utf8(String(id)),
-            };
-            await s.executeQuery(yql, params, undefined, settings);
-            deleted += 1;
-        }
-    });
-    return deleted;
-}
+export { searchPointsOneTable } from "./pointsRepo.one-table/Search.js";
+export { upsertPointsOneTable } from "./pointsRepo.one-table/Upsert.js";
+export { deletePointsOneTable, deletePointsByPathSegmentsOneTable, } from "./pointsRepo.one-table/Delete.js";

package/dist/routes/points.js

@@ -5,6 +5,19 @@ import { logger } from "../logging/logger.js";
 import { isCompilationTimeoutError } from "../ydb/client.js";
 import { scheduleExit } from "../utils/exit.js";
 export const pointsRouter = Router();
+function toQdrantScoredPoint(p) {
+    // We don't currently track per-point versions or return vectors/shard keys,
+    // but many Qdrant clients expect these fields to exist in the response.
+    return {
+        id: p.id,
+        version: 0,
+        score: p.score,
+        payload: p.payload ?? null,
+        vector: null,
+        shard_key: null,
+        order_value: null,
+    };
+}
 // Qdrant-compatible: PUT /collections/:collection/points (upsert)
 pointsRouter.put("/:collection/points", async (req, res) => {
     try {
@@ -58,13 +71,13 @@ pointsRouter.post("/:collection/points/upsert", async (req, res) => {
 });
 pointsRouter.post("/:collection/points/search", async (req, res) => {
     try {
-        const result = await searchPoints({
+        const { points } = await searchPoints({
             tenant: req.header("X-Tenant-Id") ?? undefined,
             collection: String(req.params.collection),
             apiKey: req.header("api-key") ?? undefined,
             userAgent: req.header("User-Agent") ?? undefined,
         }, req.body);
-        res.json({ status: "ok", result });
+        res.json({ status: "ok", result: points.map(toQdrantScoredPoint) });
     }
     catch (err) {
         if (err instanceof QdrantServiceError) {
@@ -84,13 +97,13 @@ pointsRouter.post("/:collection/points/search", async (req, res) => {
 // Compatibility: some clients call POST /collections/:collection/points/query
 pointsRouter.post("/:collection/points/query", async (req, res) => {
     try {
-        const result = await queryPoints({
+        const { points } = await queryPoints({
             tenant: req.header("X-Tenant-Id") ?? undefined,
             collection: String(req.params.collection),
             apiKey: req.header("api-key") ?? undefined,
             userAgent: req.header("User-Agent") ?? undefined,
         }, req.body);
-        res.json({ status: "ok", result });
+        res.json({ status: "ok", result: points.map(toQdrantScoredPoint) });
     }
     catch (err) {
         if (err instanceof QdrantServiceError) {

package/dist/server.d.ts

@@ -1,3 +1,4 @@
 import type { Request, Response } from "express";
 export declare function healthHandler(_req: Request, res: Response): Promise<void>;
+export declare function rootHandler(_req: Request, res: Response): void;
 export declare function buildServer(): import("express-serve-static-core").Express;

package/dist/server.js

@@ -22,18 +22,86 @@ export async function healthHandler(_req, res) {
         logger.error({ err }, isTimeout
             ? "YDB compilation timeout during health probe; scheduling process exit"
             : "YDB health probe failed; scheduling process exit");
-        res.status(503).json({ status: "error", error: "YDB health probe failed" });
+        res.status(503).json({
+            status: "error",
+            error: "YDB health probe failed",
+        });
         scheduleExit(1);
         return;
     }
     res.json({ status: "ok" });
 }
+export function rootHandler(_req, res) {
+    const version = process.env.npm_package_version ?? "unknown";
+    res.json({ title: "ydb-qdrant", version });
+}
 export function buildServer() {
     const app = express();
-    app.use(express.json({ limit: "20mb" }));
     app.use(requestLogger);
+    app.use(express.json({ limit: "20mb" }));
+    app.get("/", rootHandler);
     app.get("/health", healthHandler);
     app.use("/collections", collectionsRouter);
     app.use("/collections", pointsRouter);
+    app.use((err, req, res, next) => {
+        if (!isRequestAbortedError(err)) {
+            next(err);
+            return;
+        }
+        // Client closed the connection while the request body was being read.
+        // Avoid Express default handler printing a stacktrace to stderr.
+        if (res.headersSent || res.writableEnded) {
+            return;
+        }
+        res.status(400).json({ status: "error", error: "request aborted" });
+    });
+    // Catch-all error handler: avoid Express default handler printing stacktraces to stderr
+    // and provide consistent JSON error responses.
+    app.use((err, _req, res, _next) => {
+        logger.error({ err }, "Unhandled error in Express middleware");
+        void _next;
+        if (res.headersSent || res.writableEnded) {
+            return;
+        }
+        const statusCode = extractHttpStatusCode(err) ?? 500;
+        const errorMessage = err instanceof Error ? err.message : String(err);
+        res.status(statusCode).json({
+            status: "error",
+            error: errorMessage,
+        });
+    });
     return app;
 }
+function isRequestAbortedError(err) {
+    if (!err || typeof err !== "object") {
+        return false;
+    }
+    const typeValue = "type" in err && typeof err.type === "string" ? err.type : undefined;
+    if (typeValue === "request.aborted") {
+        return true;
+    }
+    if ("message" in err && typeof err.message === "string") {
+        return err.message.includes("request aborted");
+    }
+    return false;
+}
+function extractHttpStatusCode(err) {
+    if (!err || typeof err !== "object") {
+        return undefined;
+    }
+    const obj = err;
+    let statusCodeValue;
+    if (typeof obj.statusCode === "number") {
+        statusCodeValue = obj.statusCode;
+    }
+    else if (typeof obj.status === "number") {
+        statusCodeValue = obj.status;
+    }
+    if (statusCodeValue === undefined ||
+        !Number.isInteger(statusCodeValue) ||
+        statusCodeValue < 400 ||
+        statusCodeValue > 599) {
+        return undefined;
+    }
+    return statusCodeValue;
+}

package/dist/services/CollectionService.d.ts

@@ -24,6 +24,15 @@ export declare function getCollection(ctx: CollectionContextInput): Promise<{
         distance: DistanceKind;
         data_type: string;
     };
+    config: {
+        params: {
+            vectors: {
+                size: number;
+                distance: DistanceKind;
+                data_type: string;
+            };
+        };
+    };
 }>;
 export declare function deleteCollection(ctx: CollectionContextInput): Promise<{
     acknowledged: boolean;

package/dist/services/CollectionService.js

@@ -64,6 +64,15 @@ export async function getCollection(ctx) {
             distance: meta.distance,
             data_type: meta.vectorType,
         },
+        config: {
+            params: {
+                vectors: {
+                    size: meta.dimension,
+                    distance: meta.distance,
+                    data_type: meta.vectorType,
+                },
+            },
+        },
     };
 }
 export async function deleteCollection(ctx) {

package/dist/services/PointsService.d.ts

@@ -1,21 +1,14 @@
 import { type CollectionContextInput } from "./CollectionService.js";
+import type { YdbQdrantScoredPoint } from "../qdrant/QdrantRestTypes.js";
 type PointsContextInput = CollectionContextInput;
 export declare function upsertPoints(ctx: PointsContextInput, body: unknown): Promise<{
     upserted: number;
 }>;
 export declare function searchPoints(ctx: PointsContextInput, body: unknown): Promise<{
-    points: Array<{
-        id: string;
-        score: number;
-        payload?: Record<string, unknown>;
-    }>;
+    points: YdbQdrantScoredPoint[];
 }>;
 export declare function queryPoints(ctx: PointsContextInput, body: unknown): Promise<{
-    points: Array<{
-        id: string;
-        score: number;
-        payload?: Record<string, unknown>;
-    }>;
+    points: YdbQdrantScoredPoint[];
 }>;
 export declare function deletePoints(ctx: PointsContextInput, body: unknown): Promise<{
     deleted: number;