ydb-qdrant 8.1.0 → 9.0.3

package/dist/repositories/pointsRepo.one-table/Retrieve.js

@@ -0,0 +1,69 @@
+import { Types, TypedValues, withSession, createExecuteQuerySettings, } from "../../ydb/client.js";
+import { logger } from "../../logging/logger.js";
+import { computePayloadSign } from "../../utils/PayloadSign.js";
+import { isTransientYdbError, withRetry } from "../../utils/retry.js";
+const RETRIEVE_RETRY_MAX_RETRIES = 6;
+const RETRIEVE_RETRY_BASE_DELAY_MS = 250;
+const RETRIEVE_RETRY_MAX_BACKOFF_MS = 1500;
+export async function retrievePointsByIdsOneTable(tableName, ids, uid, apiKey, withPayload) {
+    const stringIds = ids.map(String);
+    const qry = `
+    DECLARE $collection AS Utf8;
+    DECLARE $ids AS List<Utf8>;
+    SELECT point_id, payload, payload_sign
+    FROM ${tableName}
+    WHERE collection = $collection AND point_id IN $ids;
+  `;
+    const res = await withRetry(async () => await withSession(async (s) => {
+        const settings = createExecuteQuerySettings();
+        return await s.executeQuery(qry, {
+            $collection: TypedValues.utf8(uid),
+            $ids: TypedValues.list(Types.UTF8, stringIds),
+        }, undefined, settings);
+    }), {
+        isTransient: isTransientYdbError,
+        maxRetries: RETRIEVE_RETRY_MAX_RETRIES,
+        baseDelayMs: RETRIEVE_RETRY_BASE_DELAY_MS,
+        maxBackoffMs: RETRIEVE_RETRY_MAX_BACKOFF_MS,
+        context: {
+            operation: "retrievePointsByIdsOneTable",
+            tableName,
+            collection: uid,
+            idCount: ids.length,
+        },
+    });
+    const rowset = res.resultSets?.[0];
+    const rows = (rowset?.rows ?? []);
+    const trimmedApiKey = apiKey.trim();
+    const out = [];
+    for (const row of rows) {
+        const id = row.items?.[0]?.textValue;
+        if (typeof id !== "string")
+            continue;
+        const payloadText = row.items?.[1]?.textValue;
+        const payloadSign = row.items?.[2]?.textValue;
+        let payload = null;
+        if (payloadText) {
+            try {
+                payload = JSON.parse(payloadText);
+            }
+            catch {
+                payload = null;
+            }
+        }
+        if (!payload || typeof payloadSign !== "string" || payloadSign === "") {
+            logger.warn({ collection: uid, pointId: id }, "retrieve: payload signature mismatch: missing payload or signature");
+            continue;
+        }
+        const expected = computePayloadSign({ apiKey: trimmedApiKey, payload });
+        if (expected !== payloadSign) {
+            logger.warn({ collection: uid, pointId: id }, "retrieve: payload signature mismatch: dropping point from results");
+            continue;
+        }
+        out.push({
+            id,
+            payload: withPayload ? payload : null,
+        });
+    }
+    return out;
+}

package/dist/repositories/pointsRepo.one-table/Search.d.ts

@@ -1,4 +1,3 @@
-import type { DistanceKind } from "../../types.js";
-import { SearchMode } from "../../config/env.js";
+import type { DistanceKind } from "../../qdrant/QdrantRestTypes.js";
 import type { YdbQdrantScoredPoint } from "../../qdrant/QdrantRestTypes.js";
-export declare function searchPointsOneTable(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number, uid: string, mode: SearchMode | undefined, overfetchMultiplier: number, filterPaths?: Array<Array<string>>): Promise<YdbQdrantScoredPoint[]>;
+export declare function searchPointsOneTable(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number, collection: string, apiKey: string, filterPaths?: Array<Array<string>>): Promise<YdbQdrantScoredPoint[]>;

package/dist/repositories/pointsRepo.one-table/Search.js

@@ -1,9 +1,11 @@
 import { Types, TypedValues, withSession, createExecuteQuerySettingsWithTimeout, } from "../../ydb/client.js";
-import { buildVectorBinaryParams } from "../../ydb/helpers.js";
-import { mapDistanceToKnnFn, mapDistanceToBitKnnFn, } from "../../utils/distance.js";
+import { mapDistanceToKnnFn } from "../../utils/distance.js";
+import { vectorToFloatBinary } from "../../utils/vectorBinary.js";
 import { logger } from "../../logging/logger.js";
-import { SearchMode, SEARCH_OPERATION_TIMEOUT_MS } from "../../config/env.js";
-import { buildPathSegmentsFilter } from "./PathSegmentsFilter.js";
+import { SEARCH_OPERATION_TIMEOUT_MS } from "../../config/env.js";
+import { buildPrefixPathSegmentsFilter } from "./PathSegmentsFilter.js";
+import { computePayloadSign } from "../../utils/PayloadSign.js";
+import { isComputePoolEnabled, isComputePoolQueueAtLimitError, runVerifySearchRows, } from "../../compute/ComputePool.js";
 function assertVectorDimension(vector, dimension, messagePrefix = "Vector dimension mismatch") {
     if (vector.length !== dimension) {
         throw new Error(`${messagePrefix}: got ${vector.length}, expected ${dimension}`);
@@ -19,114 +21,132 @@ function typedBytesOrFallback(value) {
     }
     throw new Error("ydb-sdk does not support constructing BYTES typed parameters (TypedValues.bytes/fromNative missing); cannot execute vector search");
 }
-function parseSearchRows(rows, withPayload) {
-    return rows.map((row) => {
+function parseSearchRows(rows, args) {
+    const apiKey = args.apiKey.trim();
+    const out = [];
+    for (const row of rows) {
         const id = row.items?.[0]?.textValue;
         if (typeof id !== "string") {
             throw new Error("point_id is missing in YDB search result");
         }
+        const payloadText = row.items?.[1]?.textValue;
+        const payloadSign = row.items?.[2]?.textValue;
         let payload;
-        let scoreIdx = 1;
-        if (withPayload) {
-            const payloadText = row.items?.[1]?.textValue;
-            if (payloadText) {
-                try {
-                    payload = JSON.parse(payloadText);
-                }
-                catch {
-                    payload = undefined;
-                }
+        if (payloadText) {
+            try {
+                payload = JSON.parse(payloadText);
             }
-            scoreIdx = 2;
+            catch {
+                payload = undefined;
+            }
+        }
+        if (!payload || typeof payloadSign !== "string" || payloadSign === "") {
+            logger.warn({ collection: args.collection, pointId: id }, "payload signature mismatch: missing payload or signature");
+            continue;
+        }
+        const expected = computePayloadSign({ apiKey, payload });
+        if (expected !== payloadSign) {
+            logger.warn({ collection: args.collection, pointId: id }, "payload signature mismatch: dropping point from search results");
+            continue;
+        }
+        const scoreIdx = 3;
+        const score = Number(row.items?.[scoreIdx]?.floatValue ??
+            row.items?.[scoreIdx]?.textValue);
+        const shouldReturnPayload = args.withPayload === true;
+        out.push({
+            id,
+            score,
+            ...(shouldReturnPayload && payload ? { payload } : {}),
+        });
+    }
+    return out;
+}
+async function parseSearchRowsWithWorkers(rows, args) {
+    const taskRows = rows.map((row) => {
+        const id = row.items?.[0]?.textValue;
+        if (typeof id !== "string") {
+            throw new Error("point_id is missing in YDB search result");
         }
-        const score = Number(row.items?.[scoreIdx]?.floatValue ?? row.items?.[scoreIdx]?.textValue);
-        return { id, score, ...(payload ? { payload } : {}) };
+        const payloadText = row.items?.[1]?.textValue;
+        const payloadSign = row.items?.[2]?.textValue;
+        const scoreIdx = 3;
+        const scoreFloat = row.items?.[scoreIdx]?.floatValue;
+        const scoreText = row.items?.[scoreIdx]?.textValue;
+        return {
+            pointId: id,
+            payloadText,
+            payloadSign,
+            scoreFloat,
+            scoreText,
+        };
     });
+    const result = await runVerifySearchRows({
+        collection: args.collection,
+        apiKey: args.apiKey,
+        withPayload: args.withPayload,
+        rows: taskRows,
+    });
+    for (const d of result.dropped) {
+        if (d.reason === "missing_payload_or_signature") {
+            logger.warn({ collection: args.collection, pointId: d.pointId }, "payload signature mismatch: missing payload or signature");
+            continue;
+        }
+        logger.warn({ collection: args.collection, pointId: d.pointId }, "payload signature mismatch: dropping point from search results");
+    }
+    return result.points;
+}
+async function parseSearchRowsMaybeWithWorkers(rows, args) {
+    if (!isComputePoolEnabled()) {
+        return parseSearchRows(rows, args);
+    }
+    try {
+        return await parseSearchRowsWithWorkers(rows, args);
+    }
+    catch (err) {
+        if (!isComputePoolQueueAtLimitError(err)) {
+            throw err;
+        }
+        // Backpressure: fall back to inline compute to avoid failing the request.
+        return parseSearchRows(rows, args);
+    }
 }
 function buildExactSearchQueryAndParams(args) {
     const { fn, order } = mapDistanceToKnnFn(args.distance);
-    const filter = buildPathSegmentsFilter(args.filterPaths);
+    const filter = buildPrefixPathSegmentsFilter(args.filterPaths, "path_prefix");
     const filterWhere = filter ? ` AND ${filter.whereSql}` : "";
-    const binaries = buildVectorBinaryParams(args.queryVector);
+    const qbinf = vectorToFloatBinary(args.queryVector);
     const yql = `
         DECLARE $qbinf AS String;
         DECLARE $k AS Uint32;
-        DECLARE $uid AS Utf8;
+        DECLARE $collection AS Utf8;
         ${filter?.whereParamDeclarations ?? ""}
-        SELECT point_id, ${args.withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
+        SELECT point_id, payload, payload_sign, ${fn}(embedding, $qbinf) AS score
         FROM ${args.tableName}
-        WHERE uid = $uid${filterWhere}
+        WHERE collection = $collection${filterWhere}
         ORDER BY score ${order}
         LIMIT $k;
       `;
     const params = {
         ...(filter?.whereParams ?? {}),
-        $qbinf: typedBytesOrFallback(binaries.float),
+        $qbinf: typedBytesOrFallback(qbinf),
         $k: TypedValues.uint32(args.top),
-        $uid: TypedValues.utf8(args.uid),
-    };
-    return { yql, params, modeLog: "one_table_exact_client_side_serialization" };
-}
-function buildApproxSearchQueryAndParams(args) {
-    const { fn, order } = mapDistanceToKnnFn(args.distance);
-    const { fn: bitFn, order: bitOrder } = mapDistanceToBitKnnFn(args.distance);
-    const safeTop = args.top > 0 ? args.top : 1;
-    const rawCandidateLimit = safeTop * args.overfetchMultiplier;
-    const candidateLimit = Math.max(safeTop, rawCandidateLimit);
-    const filter = buildPathSegmentsFilter(args.filterPaths);
-    const filterWhere = filter ? ` AND ${filter.whereSql}` : "";
-    const binaries = buildVectorBinaryParams(args.queryVector);
-    const yql = `
-        DECLARE $qbin_bit AS String;
-        DECLARE $qbinf AS String;
-        DECLARE $candidateLimit AS Uint32;
-        DECLARE $safeTop AS Uint32;
-        DECLARE $uid AS Utf8;
-        ${filter?.whereParamDeclarations ?? ""}
-
-        $candidates = (
-          SELECT point_id
-          FROM ${args.tableName}
-          WHERE uid = $uid AND embedding_quantized IS NOT NULL
-            ${filterWhere}
-          ORDER BY ${bitFn}(embedding_quantized, $qbin_bit) ${bitOrder}
-          LIMIT $candidateLimit
-        );
-
-        SELECT point_id, ${args.withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
-        FROM ${args.tableName}
-        WHERE uid = $uid
-          AND point_id IN $candidates
-          ${filterWhere}
-        ORDER BY score ${order}
-        LIMIT $safeTop;
-      `;
-    const params = {
-        ...(filter?.whereParams ?? {}),
-        $qbin_bit: typedBytesOrFallback(binaries.bit),
-        $qbinf: typedBytesOrFallback(binaries.float),
-        $candidateLimit: TypedValues.uint32(candidateLimit),
-        $safeTop: TypedValues.uint32(safeTop),
-        $uid: TypedValues.utf8(args.uid),
+        $collection: TypedValues.utf8(args.collection),
     };
     return {
         yql,
         params,
-        safeTop,
-        candidateLimit,
-        modeLog: "one_table_approximate_client_side_serialization",
+        modeLog: "one_table_exact_client_side_serialization",
     };
 }
-async function searchPointsOneTableExact(tableName, queryVector, top, withPayload, distance, dimension, uid, filterPaths) {
+async function searchPointsOneTableExact(tableName, queryVector, top, withPayload, apiKey, distance, dimension, collection, filterPaths) {
     assertVectorDimension(queryVector, dimension);
     const results = await withSession(async (s) => {
         const { yql, params, modeLog } = buildExactSearchQueryAndParams({
             tableName,
             queryVector,
             top,
-            withPayload,
             distance,
-            uid,
+            collection,
             filterPaths,
         });
         if (logger.isLevelEnabled("debug")) {
@@ -138,7 +158,7 @@ async function searchPointsOneTableExact(tableName, queryVector, top, withPayloa
                 mode: modeLog,
                 yql,
                 params: {
-                    uid,
+                    collection,
                     top,
                     vectorLength: queryVector.length,
                     vectorPreview: queryVector.slice(0, 3),
@@ -153,56 +173,14 @@ async function searchPointsOneTableExact(tableName, queryVector, top, withPayloa
         const rs = await s.executeQuery(yql, params, undefined, settings);
         const rowset = rs.resultSets?.[0];
         const rows = (rowset?.rows ?? []);
-        return parseSearchRows(rows, withPayload);
-    });
-    return results;
-}
-async function searchPointsOneTableApproximate(tableName, queryVector, top, withPayload, distance, dimension, uid, overfetchMultiplier, filterPaths) {
-    assertVectorDimension(queryVector, dimension);
-    const results = await withSession(async (s) => {
-        const { yql, params, safeTop, candidateLimit, modeLog } = buildApproxSearchQueryAndParams({
-            tableName,
-            queryVector,
-            top,
+        return await parseSearchRowsMaybeWithWorkers(rows, {
+            collection,
             withPayload,
-            distance,
-            uid,
-            overfetchMultiplier,
-            filterPaths,
+            apiKey,
         });
-        if (logger.isLevelEnabled("debug")) {
-            logger.debug({
-                tableName,
-                distance,
-                top,
-                safeTop,
-                candidateLimit,
-                mode: modeLog,
-                yql,
-                params: {
-                    uid,
-                    safeTop,
-                    candidateLimit,
-                    vectorLength: queryVector.length,
-                    vectorPreview: queryVector.slice(0, 3),
-                },
-            }, "one_table search (approximate): executing YQL");
-        }
-        const settings = createExecuteQuerySettingsWithTimeout({
-            keepInCache: true,
-            idempotent: true,
-            timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
-        });
-        const rs = await s.executeQuery(yql, params, undefined, settings);
-        const rowset = rs.resultSets?.[0];
-        const rows = (rowset?.rows ?? []);
-        return parseSearchRows(rows, withPayload);
     });
     return results;
 }
-export async function searchPointsOneTable(tableName, queryVector, top, withPayload, distance, dimension, uid, mode, overfetchMultiplier, filterPaths) {
-    if (mode === SearchMode.Exact) {
-        return await searchPointsOneTableExact(tableName, queryVector, top, withPayload, distance, dimension, uid, filterPaths);
-    }
-    return await searchPointsOneTableApproximate(tableName, queryVector, top, withPayload, distance, dimension, uid, overfetchMultiplier, filterPaths);
+export async function searchPointsOneTable(tableName, queryVector, top, withPayload, distance, dimension, collection, apiKey, filterPaths) {
+    return await searchPointsOneTableExact(tableName, queryVector, top, withPayload, apiKey, distance, dimension, collection, filterPaths);
 }

package/dist/repositories/pointsRepo.one-table/Upsert.d.ts

@@ -1,2 +1,2 @@
-import type { UpsertPoint } from "../../types.js";
-export declare function upsertPointsOneTable(tableName: string, points: UpsertPoint[], dimension: number, uid: string): Promise<number>;
+import type { UpsertPoint } from "../../qdrant/Requests.js";
+export declare function upsertPointsOneTable(tableName: string, points: UpsertPoint[], dimension: number, collection: string, apiKey: string): Promise<number>;