ydb-qdrant 6.0.0 → 8.1.0

package/dist/repositories/pointsRepo.one-table/Search.js

@@ -0,0 +1,208 @@
+import { Types, TypedValues, withSession, createExecuteQuerySettingsWithTimeout, } from "../../ydb/client.js";
+import { buildVectorBinaryParams } from "../../ydb/helpers.js";
+import { mapDistanceToKnnFn, mapDistanceToBitKnnFn, } from "../../utils/distance.js";
+import { logger } from "../../logging/logger.js";
+import { SearchMode, SEARCH_OPERATION_TIMEOUT_MS } from "../../config/env.js";
+import { buildPathSegmentsFilter } from "./PathSegmentsFilter.js";
+function assertVectorDimension(vector, dimension, messagePrefix = "Vector dimension mismatch") {
+    if (vector.length !== dimension) {
+        throw new Error(`${messagePrefix}: got ${vector.length}, expected ${dimension}`);
+    }
+}
+function typedBytesOrFallback(value) {
+    const typedValuesCompat = TypedValues;
+    if (typeof typedValuesCompat.bytes === "function") {
+        return typedValuesCompat.bytes(value);
+    }
+    if (typeof typedValuesCompat.fromNative === "function") {
+        return typedValuesCompat.fromNative(Types.BYTES, value);
+    }
+    throw new Error("ydb-sdk does not support constructing BYTES typed parameters (TypedValues.bytes/fromNative missing); cannot execute vector search");
+}
+function parseSearchRows(rows, withPayload) {
+    return rows.map((row) => {
+        const id = row.items?.[0]?.textValue;
+        if (typeof id !== "string") {
+            throw new Error("point_id is missing in YDB search result");
+        }
+        let payload;
+        let scoreIdx = 1;
+        if (withPayload) {
+            const payloadText = row.items?.[1]?.textValue;
+            if (payloadText) {
+                try {
+                    payload = JSON.parse(payloadText);
+                }
+                catch {
+                    payload = undefined;
+                }
+            }
+            scoreIdx = 2;
+        }
+        const score = Number(row.items?.[scoreIdx]?.floatValue ?? row.items?.[scoreIdx]?.textValue);
+        return { id, score, ...(payload ? { payload } : {}) };
+    });
+}
+function buildExactSearchQueryAndParams(args) {
+    const { fn, order } = mapDistanceToKnnFn(args.distance);
+    const filter = buildPathSegmentsFilter(args.filterPaths);
+    const filterWhere = filter ? ` AND ${filter.whereSql}` : "";
+    const binaries = buildVectorBinaryParams(args.queryVector);
+    const yql = `
+        DECLARE $qbinf AS String;
+        DECLARE $k AS Uint32;
+        DECLARE $uid AS Utf8;
+        ${filter?.whereParamDeclarations ?? ""}
+        SELECT point_id, ${args.withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
+        FROM ${args.tableName}
+        WHERE uid = $uid${filterWhere}
+        ORDER BY score ${order}
+        LIMIT $k;
+      `;
+    const params = {
+        ...(filter?.whereParams ?? {}),
+        $qbinf: typedBytesOrFallback(binaries.float),
+        $k: TypedValues.uint32(args.top),
+        $uid: TypedValues.utf8(args.uid),
+    };
+    return { yql, params, modeLog: "one_table_exact_client_side_serialization" };
+}
+function buildApproxSearchQueryAndParams(args) {
+    const { fn, order } = mapDistanceToKnnFn(args.distance);
+    const { fn: bitFn, order: bitOrder } = mapDistanceToBitKnnFn(args.distance);
+    const safeTop = args.top > 0 ? args.top : 1;
+    const rawCandidateLimit = safeTop * args.overfetchMultiplier;
+    const candidateLimit = Math.max(safeTop, rawCandidateLimit);
+    const filter = buildPathSegmentsFilter(args.filterPaths);
+    const filterWhere = filter ? ` AND ${filter.whereSql}` : "";
+    const binaries = buildVectorBinaryParams(args.queryVector);
+    const yql = `
+        DECLARE $qbin_bit AS String;
+        DECLARE $qbinf AS String;
+        DECLARE $candidateLimit AS Uint32;
+        DECLARE $safeTop AS Uint32;
+        DECLARE $uid AS Utf8;
+        ${filter?.whereParamDeclarations ?? ""}
+
+        $candidates = (
+          SELECT point_id
+          FROM ${args.tableName}
+          WHERE uid = $uid AND embedding_quantized IS NOT NULL
+            ${filterWhere}
+          ORDER BY ${bitFn}(embedding_quantized, $qbin_bit) ${bitOrder}
+          LIMIT $candidateLimit
+        );
+
+        SELECT point_id, ${args.withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
+        FROM ${args.tableName}
+        WHERE uid = $uid
+          AND point_id IN $candidates
+          ${filterWhere}
+        ORDER BY score ${order}
+        LIMIT $safeTop;
+      `;
+    const params = {
+        ...(filter?.whereParams ?? {}),
+        $qbin_bit: typedBytesOrFallback(binaries.bit),
+        $qbinf: typedBytesOrFallback(binaries.float),
+        $candidateLimit: TypedValues.uint32(candidateLimit),
+        $safeTop: TypedValues.uint32(safeTop),
+        $uid: TypedValues.utf8(args.uid),
+    };
+    return {
+        yql,
+        params,
+        safeTop,
+        candidateLimit,
+        modeLog: "one_table_approximate_client_side_serialization",
+    };
+}
+async function searchPointsOneTableExact(tableName, queryVector, top, withPayload, distance, dimension, uid, filterPaths) {
+    assertVectorDimension(queryVector, dimension);
+    const results = await withSession(async (s) => {
+        const { yql, params, modeLog } = buildExactSearchQueryAndParams({
+            tableName,
+            queryVector,
+            top,
+            withPayload,
+            distance,
+            uid,
+            filterPaths,
+        });
+        if (logger.isLevelEnabled("debug")) {
+            logger.debug({
+                tableName,
+                distance,
+                top,
+                withPayload,
+                mode: modeLog,
+                yql,
+                params: {
+                    uid,
+                    top,
+                    vectorLength: queryVector.length,
+                    vectorPreview: queryVector.slice(0, 3),
+                },
+            }, "one_table search (exact): executing YQL");
+        }
+        const settings = createExecuteQuerySettingsWithTimeout({
+            keepInCache: true,
+            idempotent: true,
+            timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
+        });
+        const rs = await s.executeQuery(yql, params, undefined, settings);
+        const rowset = rs.resultSets?.[0];
+        const rows = (rowset?.rows ?? []);
+        return parseSearchRows(rows, withPayload);
+    });
+    return results;
+}
+async function searchPointsOneTableApproximate(tableName, queryVector, top, withPayload, distance, dimension, uid, overfetchMultiplier, filterPaths) {
+    assertVectorDimension(queryVector, dimension);
+    const results = await withSession(async (s) => {
+        const { yql, params, safeTop, candidateLimit, modeLog } = buildApproxSearchQueryAndParams({
+            tableName,
+            queryVector,
+            top,
+            withPayload,
+            distance,
+            uid,
+            overfetchMultiplier,
+            filterPaths,
+        });
+        if (logger.isLevelEnabled("debug")) {
+            logger.debug({
+                tableName,
+                distance,
+                top,
+                safeTop,
+                candidateLimit,
+                mode: modeLog,
+                yql,
+                params: {
+                    uid,
+                    safeTop,
+                    candidateLimit,
+                    vectorLength: queryVector.length,
+                    vectorPreview: queryVector.slice(0, 3),
+                },
+            }, "one_table search (approximate): executing YQL");
+        }
+        const settings = createExecuteQuerySettingsWithTimeout({
+            keepInCache: true,
+            idempotent: true,
+            timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
+        });
+        const rs = await s.executeQuery(yql, params, undefined, settings);
+        const rowset = rs.resultSets?.[0];
+        const rows = (rowset?.rows ?? []);
+        return parseSearchRows(rows, withPayload);
+    });
+    return results;
+}
+export async function searchPointsOneTable(tableName, queryVector, top, withPayload, distance, dimension, uid, mode, overfetchMultiplier, filterPaths) {
+    if (mode === SearchMode.Exact) {
+        return await searchPointsOneTableExact(tableName, queryVector, top, withPayload, distance, dimension, uid, filterPaths);
+    }
+    return await searchPointsOneTableApproximate(tableName, queryVector, top, withPayload, distance, dimension, uid, overfetchMultiplier, filterPaths);
+}

package/dist/repositories/pointsRepo.one-table/Upsert.d.ts

@@ -0,0 +1,2 @@
+import type { UpsertPoint } from "../../types.js";
+export declare function upsertPointsOneTable(tableName: string, points: UpsertPoint[], dimension: number, uid: string): Promise<number>;

package/dist/repositories/pointsRepo.one-table/Upsert.js

@@ -0,0 +1,85 @@
+import { TypedValues, Types, withSession, Ydb as YdbRuntime, createBulkUpsertSettingsWithTimeout, } from "../../ydb/client.js";
+import { buildVectorBinaryParams } from "../../ydb/helpers.js";
+import { withRetry, isTransientYdbError } from "../../utils/retry.js";
+import { UPSERT_BATCH_SIZE } from "../../ydb/schema.js";
+import { UPSERT_OPERATION_TIMEOUT_MS } from "../../config/env.js";
+import { logger } from "../../logging/logger.js";
+function assertPointVectorsDimension(args) {
+    for (const p of args.points) {
+        const id = String(p.id);
+        if (p.vector.length !== args.dimension) {
+            const previewLength = Math.min(16, p.vector.length);
+            const vectorPreview = previewLength > 0 ? p.vector.slice(0, previewLength) : [];
+            logger.warn({
+                tableName: args.tableName,
+                uid: args.uid,
+                pointId: id,
+                vectorLen: p.vector.length,
+                expectedDimension: args.dimension,
+                vectorPreview,
+            }, "upsertPointsOneTable: vector dimension mismatch");
+            throw new Error(`Vector dimension mismatch for id=${id}: got ${p.vector.length}, expected ${args.dimension}`);
+        }
+    }
+}
+function buildBulkUpsertRowsValue(args) {
+    const rowType = Types.struct({
+        uid: Types.UTF8,
+        point_id: Types.UTF8,
+        embedding: Types.BYTES,
+        embedding_quantized: Types.BYTES,
+        payload: Types.JSON_DOCUMENT,
+    });
+    return TypedValues.list(rowType, args.batch.map((p) => {
+        const binaries = buildVectorBinaryParams(p.vector);
+        return {
+            uid: args.uid,
+            point_id: String(p.id),
+            embedding: binaries.float,
+            embedding_quantized: binaries.bit,
+            payload: JSON.stringify(p.payload ?? {}),
+        };
+    }));
+}
+export async function upsertPointsOneTable(tableName, points, dimension, uid) {
+    if (!tableName) {
+        throw new Error("bulkUpsert: tableName is empty");
+    }
+    assertPointVectorsDimension({ tableName, uid, points, dimension });
+    let upserted = 0;
+    await withSession(async (s) => {
+        const bulkSettings = createBulkUpsertSettingsWithTimeout({
+            timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
+        });
+        for (let i = 0; i < points.length; i += UPSERT_BATCH_SIZE) {
+            const batch = points.slice(i, i + UPSERT_BATCH_SIZE);
+            const rowsValue = buildBulkUpsertRowsValue({
+                uid,
+                batch,
+            });
+            if (logger.isLevelEnabled("debug")) {
+                logger.debug({
+                    tableName,
+                    mode: "one_table_bulk_upsert_client_side_serialization",
+                    batchSize: batch.length,
+                    params: {
+                        rows: batch.map((p) => ({
+                            uid,
+                            point_id: String(p.id),
+                            vectorLength: p.vector.length,
+                            vectorPreview: p.vector.slice(0, 3),
+                            payload: p.payload ?? {},
+                        })),
+                    },
+                }, "one_table upsert: executing BulkUpsert");
+            }
+            const typedRows = YdbRuntime.TypedValue.create(rowsValue);
+            await withRetry(() => s.bulkUpsert(tableName, typedRows, bulkSettings), {
+                isTransient: isTransientYdbError,
+                context: { tableName, batchSize: batch.length, mode: "bulkUpsert" },
+            });
+            upserted += batch.length;
+        }
+    });
+    return upserted;
+}

package/dist/repositories/pointsRepo.one-table.d.ts

@@ -1,13 +1,3 @@
-import type { DistanceKind } from "../types";
-import { SearchMode } from "../config/env.js";
-export declare function upsertPointsOneTable(tableName: string, points: Array<{
-    id: string | number;
-    vector: number[];
-    payload?: Record<string, unknown>;
-}>, dimension: number, uid: string): Promise<number>;
-export declare function searchPointsOneTable(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number, uid: string, mode: SearchMode | undefined, overfetchMultiplier: number): Promise<Array<{
-    id: string;
-    score: number;
-    payload?: Record<string, unknown>;
-}>>;
-export declare function deletePointsOneTable(tableName: string, ids: Array<string | number>, uid: string): Promise<number>;
+export { searchPointsOneTable } from "./pointsRepo.one-table/Search.js";
+export { upsertPointsOneTable } from "./pointsRepo.one-table/Upsert.js";
+export { deletePointsOneTable, deletePointsByPathSegmentsOneTable, } from "./pointsRepo.one-table/Delete.js";