ydb-qdrant 5.2.1 → 7.0.1

package/dist/repositories/pointsRepo.one-table/Delete.js

@@ -0,0 +1,111 @@
+import { withSession } from "../../ydb/client.js";
+import { buildPathSegmentsWhereClause } from "./PathSegmentsFilter.js";
+import { Uint32, Utf8 } from "@ydbjs/value/primitive";
+import { withRetry, isTransientYdbError } from "../../utils/retry.js";
+import { UPSERT_OPERATION_TIMEOUT_MS } from "../../config/env.js";
+import { attachQueryDiagnostics } from "../../ydb/QueryDiagnostics.js";
+const DELETE_FILTER_SELECT_BATCH_SIZE = 1000;
+export async function deletePointsOneTable(tableName, ids, uid) {
+    let deleted = 0;
+    await withSession(async (sql, signal) => {
+        for (const id of ids) {
+            await withRetry(async () => {
+                await attachQueryDiagnostics(sql `
+            DELETE FROM ${sql.identifier(tableName)}
+            WHERE uid = $uid AND point_id = $id;
+          `, {
+                    operation: "deletePointsOneTable",
+                    tableName,
+                    uid,
+                    pointId: String(id),
+                })
+                    .parameter("uid", new Utf8(uid))
+                    .parameter("id", new Utf8(String(id)))
+                    .idempotent(true)
+                    .timeout(UPSERT_OPERATION_TIMEOUT_MS)
+                    .signal(signal);
+            }, {
+                isTransient: isTransientYdbError,
+                context: {
+                    operation: "deletePointsOneTable",
+                    tableName,
+                    uid,
+                    pointId: String(id),
+                },
+            });
+            deleted += 1;
+        }
+    });
+    return deleted;
+}
+function toCount(value) {
+    if (typeof value === "number" && Number.isFinite(value))
+        return value;
+    if (typeof value === "bigint") {
+        const n = Number(value);
+        return Number.isFinite(n) ? n : 0;
+    }
+    if (typeof value === "string") {
+        const n = Number(value);
+        return Number.isFinite(n) ? n : 0;
+    }
+    return 0;
+}
+export async function deletePointsByPathSegmentsOneTable(tableName, uid, paths) {
+    if (paths.length === 0) {
+        return 0;
+    }
+    const { whereSql, params: whereParams } = buildPathSegmentsWhereClause(paths);
+    let deleted = 0;
+    await withSession(async (sql, signal) => {
+        // Best-effort loop: stop when there are no more matching rows.
+        // Use limited batches to avoid per-operation buffer limits.
+        while (true) {
+            const [rows] = await withRetry(async () => {
+                let q = attachQueryDiagnostics(sql `
+            $to_delete = (
+              SELECT uid, point_id
+              FROM ${sql.identifier(tableName)}
+              WHERE uid = $uid AND ${sql.unsafe(whereSql)}
+              LIMIT $limit
+            );
+
+            DELETE FROM ${sql.identifier(tableName)} ON
+            SELECT uid, point_id FROM $to_delete;
+
+            SELECT COUNT(*) AS deleted FROM $to_delete;
+          `, {
+                    operation: "deletePointsByPathSegmentsOneTable",
+                    tableName,
+                    uid,
+                    batchLimit: DELETE_FILTER_SELECT_BATCH_SIZE,
+                    pathsCount: paths.length,
+                })
+                    .idempotent(true)
+                    .timeout(UPSERT_OPERATION_TIMEOUT_MS)
+                    .signal(signal)
+                    .parameter("uid", new Utf8(uid))
+                    .parameter("limit", new Uint32(DELETE_FILTER_SELECT_BATCH_SIZE));
+                for (const [key, value] of Object.entries(whereParams)) {
+                    q = q.parameter(key, value);
+                }
+                return await q;
+            }, {
+                isTransient: isTransientYdbError,
+                context: {
+                    operation: "deletePointsByPathSegmentsOneTable",
+                    tableName,
+                    uid,
+                    batchLimit: DELETE_FILTER_SELECT_BATCH_SIZE,
+                    pathsCount: paths.length,
+                },
+            });
+            const batchDeleted = toCount(rows[0]?.deleted);
+            if (batchDeleted <= 0) {
+                break;
+            }
+            deleted += batchDeleted;
+        }
+    });
+    return deleted;
+}

package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.d.ts

@@ -0,0 +1,11 @@
+import type { Value } from "@ydbjs/value";
+type QueryParams = Record<string, Value>;
+export declare function buildPathSegmentsWhereClause(paths: Array<Array<string>>): {
+    whereSql: string;
+    params: QueryParams;
+};
+export declare function buildPathSegmentsFilter(paths: Array<Array<string>> | undefined): {
+    whereSql: string;
+    whereParams: QueryParams;
+} | undefined;
+export {};

package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.js

@@ -0,0 +1,32 @@
+import { Utf8 } from "@ydbjs/value/primitive";
+export function buildPathSegmentsWhereClause(paths) {
+    const params = {};
+    const orGroups = [];
+    for (let pIdx = 0; pIdx < paths.length; pIdx += 1) {
+        const segs = paths[pIdx];
+        if (segs.length === 0) {
+            throw new Error("delete-by-filter: empty path segments");
+        }
+        const andParts = [];
+        for (let sIdx = 0; sIdx < segs.length; sIdx += 1) {
+            const paramName = `$p${pIdx}_${sIdx}`;
+            // payload is JsonDocument; JSON_VALUE supports JsonPath access.
+            // Security: path segment values are always bound as parameters (see `params[paramName]`)
+            // and MUST NOT be interpolated into `whereSql`. The only dynamic part in the SQL text
+            // below is the numeric segment index (sIdx) and the internal parameter name.
+            andParts.push(`JSON_VALUE(payload, '$.pathSegments."${sIdx}"') = ${paramName}`);
+            params[paramName] = new Utf8(segs[sIdx]);
+        }
+        orGroups.push(`(${andParts.join(" AND ")})`);
+    }
+    return {
+        whereSql: orGroups.length === 1 ? orGroups[0] : `(${orGroups.join(" OR ")})`,
+        params,
+    };
+}
+export function buildPathSegmentsFilter(paths) {
+    if (!paths || paths.length === 0)
+        return undefined;
+    const { whereSql, params: whereParams } = buildPathSegmentsWhereClause(paths);
+    return { whereSql, whereParams };
+}

package/dist/repositories/pointsRepo.one-table/Search/Approximate.d.ts

@@ -0,0 +1,18 @@
+import type { DistanceKind } from "../../../types";
+import type { QdrantPayload } from "../../../qdrant/QdrantTypes.js";
+export declare function searchPointsOneTableApproximate(args: {
+    tableName: string;
+    queryVector: number[];
+    top: number;
+    withPayload: boolean | undefined;
+    distance: DistanceKind;
+    dimension: number;
+    uid: string;
+    overfetchMultiplier: number;
+    timeoutMs: number;
+    filterPaths?: Array<Array<string>>;
+}): Promise<Array<{
+    id: string;
+    score: number;
+    payload?: QdrantPayload;
+}>>;

package/dist/repositories/pointsRepo.one-table/Search/Approximate.js

@@ -0,0 +1,119 @@
+import { withSession } from "../../../ydb/client.js";
+import { buildVectorBinaryParams } from "../../../ydb/helpers.js";
+import { Bytes, Uint32, Utf8 } from "@ydbjs/value/primitive";
+import { mapDistanceToBitKnnFn, mapDistanceToKnnFn, } from "../../../utils/distance.js";
+import { buildPathSegmentsFilter } from "../PathSegmentsFilter.js";
+import { attachQueryDiagnostics } from "../../../ydb/QueryDiagnostics.js";
+import { isRecord } from "../../../utils/typeGuards.js";
+function assertVectorDimension(vector, dimension, messagePrefix = "Vector dimension mismatch") {
+    if (vector.length !== dimension) {
+        throw new Error(`${messagePrefix}: got ${vector.length}, expected ${dimension}`);
+    }
+}
+function parsePayloadJson(payloadText) {
+    if (isRecord(payloadText)) {
+        return payloadText;
+    }
+    if (typeof payloadText !== "string" || payloadText.length === 0) {
+        return undefined;
+    }
+    try {
+        return JSON.parse(payloadText);
+    }
+    catch {
+        return undefined;
+    }
+}
+function buildApproxSearchParams(args) {
+    const safeTop = args.top > 0 ? args.top : 1;
+    const rawCandidateLimit = safeTop * args.overfetchMultiplier;
+    const candidateLimit = Math.max(safeTop, rawCandidateLimit);
+    const filter = buildPathSegmentsFilter(args.filterPaths);
+    const binaries = buildVectorBinaryParams(args.queryVector);
+    return {
+        params: {
+            ...(filter?.whereParams ?? {}),
+            $qbin_bit: new Bytes(binaries.bit),
+            $qbinf: new Bytes(binaries.float),
+            $candidateLimit: new Uint32(candidateLimit),
+            $safeTop: new Uint32(safeTop),
+            $uid: new Utf8(args.uid),
+        },
+        filterWhereSql: filter?.whereSql,
+    };
+}
+export async function searchPointsOneTableApproximate(args) {
+    assertVectorDimension(args.queryVector, args.dimension);
+    return await withSession(async (sql, signal) => {
+        const { fn, order } = mapDistanceToKnnFn(args.distance);
+        const { fn: bitFn, order: bitOrder } = mapDistanceToBitKnnFn(args.distance);
+        const { params, filterWhereSql } = buildApproxSearchParams({
+            queryVector: args.queryVector,
+            top: args.top,
+            uid: args.uid,
+            overfetchMultiplier: args.overfetchMultiplier,
+            filterPaths: args.filterPaths,
+        });
+        let payloadColumn;
+        if (args.withPayload) {
+            payloadColumn = sql.unsafe(", payload");
+        }
+        else {
+            payloadColumn = sql.unsafe("");
+        }
+        let filterClause;
+        if (filterWhereSql) {
+            filterClause = sql.unsafe(` AND ${filterWhereSql}`);
+        }
+        else {
+            filterClause = sql.unsafe("");
+        }
+        const baseQuery = sql `
+      $candidates = (
+        SELECT point_id
+        FROM ${sql.identifier(args.tableName)}
+        WHERE uid = $uid
+          AND embedding_quantized IS NOT NULL${filterClause}
+        ORDER BY ${sql.unsafe(bitFn)}(embedding_quantized, $qbin_bit) ${sql.unsafe(bitOrder)}
+        LIMIT $candidateLimit
+      );
+
+      SELECT
+        point_id${payloadColumn},
+        ${sql.unsafe(fn)}(embedding, $qbinf) AS score
+      FROM ${sql.identifier(args.tableName)}
+      WHERE uid = $uid
+        AND point_id IN $candidates${filterClause}
+      ORDER BY score ${sql.unsafe(order)}
+      LIMIT $safeTop;
+    `;
+        let q = attachQueryDiagnostics(baseQuery, {
+            operation: "searchPointsOneTableApproximate",
+            tableName: args.tableName,
+            uid: args.uid,
+            distance: args.distance,
+            withPayload: Boolean(args.withPayload),
+            overfetchMultiplier: args.overfetchMultiplier,
+        })
+            .idempotent(true)
+            .timeout(args.timeoutMs)
+            .signal(signal);
+        for (const [key, value] of Object.entries(params)) {
+            q = q.parameter(key, value);
+        }
+        const [rows] = await q;
+        return rows.map((r) => {
+            if (!r.point_id) {
+                throw new Error("point_id is missing in YDB search result");
+            }
+            const payload = args.withPayload
+                ? parsePayloadJson(r.payload)
+                : undefined;
+            return {
+                id: r.point_id,
+                score: Number(r.score),
+                ...(payload ? { payload } : {}),
+            };
+        });
+    });
+}

package/dist/repositories/pointsRepo.one-table/Search/Exact.d.ts

@@ -0,0 +1,17 @@
+import type { DistanceKind } from "../../../types";
+import type { QdrantPayload } from "../../../qdrant/QdrantTypes.js";
+export declare function searchPointsOneTableExact(args: {
+    tableName: string;
+    queryVector: number[];
+    top: number;
+    withPayload: boolean | undefined;
+    distance: DistanceKind;
+    dimension: number;
+    uid: string;
+    timeoutMs: number;
+    filterPaths?: Array<Array<string>>;
+}): Promise<Array<{
+    id: string;
+    score: number;
+    payload?: QdrantPayload;
+}>>;

package/dist/repositories/pointsRepo.one-table/Search/Exact.js

@@ -0,0 +1,101 @@
+import { withSession } from "../../../ydb/client.js";
+import { buildVectorBinaryParams } from "../../../ydb/helpers.js";
+import { Bytes, Uint32, Utf8 } from "@ydbjs/value/primitive";
+import { mapDistanceToKnnFn } from "../../../utils/distance.js";
+import { buildPathSegmentsFilter } from "../PathSegmentsFilter.js";
+import { attachQueryDiagnostics } from "../../../ydb/QueryDiagnostics.js";
+import { isRecord } from "../../../utils/typeGuards.js";
+function assertVectorDimension(vector, dimension, messagePrefix = "Vector dimension mismatch") {
+    if (vector.length !== dimension) {
+        throw new Error(`${messagePrefix}: got ${vector.length}, expected ${dimension}`);
+    }
+}
+function parsePayloadJson(payloadText) {
+    if (isRecord(payloadText)) {
+        return payloadText;
+    }
+    if (typeof payloadText !== "string" || payloadText.length === 0) {
+        return undefined;
+    }
+    try {
+        return JSON.parse(payloadText);
+    }
+    catch {
+        return undefined;
+    }
+}
+function buildExactSearchParams(args) {
+    const filter = buildPathSegmentsFilter(args.filterPaths);
+    const binaries = buildVectorBinaryParams(args.queryVector);
+    return {
+        params: {
+            ...(filter?.whereParams ?? {}),
+            $qbinf: new Bytes(binaries.float),
+            $k: new Uint32(args.top),
+            $uid: new Utf8(args.uid),
+        },
+        filterWhereSql: filter?.whereSql,
+    };
+}
+export async function searchPointsOneTableExact(args) {
+    assertVectorDimension(args.queryVector, args.dimension);
+    return await withSession(async (sql, signal) => {
+        const { fn, order } = mapDistanceToKnnFn(args.distance);
+        const { params, filterWhereSql } = buildExactSearchParams({
+            queryVector: args.queryVector,
+            top: args.top,
+            uid: args.uid,
+            filterPaths: args.filterPaths,
+        });
+        let payloadColumn;
+        if (args.withPayload) {
+            payloadColumn = sql.unsafe(", payload");
+        }
+        else {
+            payloadColumn = sql.unsafe("");
+        }
+        let filterClause;
+        if (filterWhereSql) {
+            filterClause = sql.unsafe(` AND ${filterWhereSql}`);
+        }
+        else {
+            filterClause = sql.unsafe("");
+        }
+        const baseQuery = sql `
+      SELECT
+        point_id${payloadColumn},
+        ${sql.unsafe(fn)}(embedding, $qbinf) AS score
+      FROM ${sql.identifier(args.tableName)}
+      WHERE uid = $uid${filterClause}
+      ORDER BY score ${sql.unsafe(order)}
+      LIMIT $k;
+    `;
+        let q = attachQueryDiagnostics(baseQuery, {
+            operation: "searchPointsOneTableExact",
+            tableName: args.tableName,
+            uid: args.uid,
+            distance: args.distance,
+            withPayload: Boolean(args.withPayload),
+        })
+            .idempotent(true)
+            .timeout(args.timeoutMs)
+            .signal(signal);
+        for (const [key, value] of Object.entries(params)) {
+            q = q.parameter(key, value);
+        }
+        const [rows] = await q;
+        return rows.map((r) => {
+            if (!r.point_id) {
+                throw new Error("point_id is missing in YDB search result");
+            }
+            const payload = args.withPayload
+                ? parsePayloadJson(r.payload)
+                : undefined;
+            return {
+                id: r.point_id,
+                score: Number(r.score),
+                ...(payload ? { payload } : {}),
+            };
+        });
+    });
+}

package/dist/repositories/pointsRepo.one-table/Search/index.d.ts

@@ -0,0 +1,8 @@
+import type { DistanceKind } from "../../../types";
+import { SearchMode } from "../../../config/env.js";
+import type { QdrantPayload } from "../../../qdrant/QdrantTypes.js";
+export declare function searchPointsOneTable(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number, uid: string, mode: SearchMode | undefined, overfetchMultiplier: number, filterPaths?: Array<Array<string>>): Promise<Array<{
+    id: string;
+    score: number;
+    payload?: QdrantPayload;
+}>>;

package/dist/repositories/pointsRepo.one-table/Search/index.js

@@ -0,0 +1,30 @@
+import { SearchMode, SEARCH_OPERATION_TIMEOUT_MS, } from "../../../config/env.js";
+import { searchPointsOneTableExact } from "./Exact.js";
+import { searchPointsOneTableApproximate } from "./Approximate.js";
+export async function searchPointsOneTable(tableName, queryVector, top, withPayload, distance, dimension, uid, mode, overfetchMultiplier, filterPaths) {
+    if (mode === SearchMode.Exact) {
+        return await searchPointsOneTableExact({
+            tableName,
+            queryVector,
+            top,
+            withPayload,
+            distance,
+            dimension,
+            uid,
+            timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
+            filterPaths,
+        });
+    }
+    return await searchPointsOneTableApproximate({
+        tableName,
+        queryVector,
+        top,
+        withPayload,
+        distance,
+        dimension,
+        uid,
+        overfetchMultiplier,
+        timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
+        filterPaths,
+    });
+}

package/dist/repositories/pointsRepo.one-table/Upsert.d.ts

@@ -0,0 +1,2 @@
+import type { QdrantPointStructDense } from "../../qdrant/QdrantTypes.js";
+export declare function upsertPointsOneTable(tableName: string, points: QdrantPointStructDense[], dimension: number, uid: string): Promise<number>;

package/dist/repositories/pointsRepo.one-table/Upsert.js

@@ -0,0 +1,100 @@
+import { getAbortErrorCause, isTimeoutAbortError } from "../../ydb/client.js";
+import { buildVectorBinaryParams } from "../../ydb/helpers.js";
+import { UPSERT_BATCH_SIZE } from "../../ydb/schema.js";
+import { UPSERT_OPERATION_TIMEOUT_MS } from "../../config/env.js";
+import { logger } from "../../logging/logger.js";
+import { withRetry, isTransientYdbError } from "../../utils/retry.js";
+import { bulkUpsertRowsOnce } from "../../ydb/bulkUpsert.js";
+import { Bytes, JsonDocument, Utf8 } from "@ydbjs/value/primitive";
+import { List } from "@ydbjs/value/list";
+import { Struct } from "@ydbjs/value/struct";
+function assertPointVectorsDimension(args) {
+    for (const p of args.points) {
+        const id = String(p.id);
+        if (p.vector.length !== args.dimension) {
+            const previewLength = Math.min(16, p.vector.length);
+            const vectorPreview = previewLength > 0 ? p.vector.slice(0, previewLength) : [];
+            logger.warn({
+                tableName: args.tableName,
+                uid: args.uid,
+                pointId: id,
+                vectorLen: p.vector.length,
+                expectedDimension: args.dimension,
+                vectorPreview,
+            }, "upsertPointsOneTable: vector dimension mismatch");
+            throw new Error(`Vector dimension mismatch for id=${id}: got ${p.vector.length}, expected ${args.dimension}`);
+        }
+    }
+}
+function buildUpsertQueryAndParams(args) {
+    const rows = args.batch.map((p) => {
+        const binaries = buildVectorBinaryParams(p.vector);
+        return {
+            uid: new Utf8(args.uid),
+            point_id: new Utf8(String(p.id)),
+            embedding: new Bytes(binaries.float),
+            embedding_quantized: new Bytes(binaries.bit),
+            payload: new JsonDocument(JSON.stringify(p.payload ?? {})),
+        };
+    });
+    const rowsValue = new List(...rows.map((row) => new Struct({
+        uid: row.uid,
+        point_id: row.point_id,
+        embedding: row.embedding,
+        embedding_quantized: row.embedding_quantized,
+        payload: row.payload,
+    })));
+    return {
+        rowsValue,
+    };
+}
+export async function upsertPointsOneTable(tableName, points, dimension, uid) {
+    assertPointVectorsDimension({ tableName, uid, points, dimension });
+    let upserted = 0;
+    for (let i = 0; i < points.length; i += UPSERT_BATCH_SIZE) {
+        const batch = points.slice(i, i + UPSERT_BATCH_SIZE);
+        const { rowsValue } = buildUpsertQueryAndParams({
+            tableName,
+            uid,
+            batch,
+        });
+        await withRetry(async () => {
+            const startedAtMs = Date.now();
+            try {
+                await bulkUpsertRowsOnce({
+                    tableName,
+                    rowsValue,
+                    timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
+                });
+            }
+            catch (err) {
+                const durationMs = Date.now() - startedAtMs;
+                if (err instanceof Error && err.name === "AbortError") {
+                    logger.warn({
+                        tableName,
+                        uid,
+                        batchStart: i,
+                        batchSize: batch.length,
+                        timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
+                        durationMs,
+                        err,
+                        errCause: getAbortErrorCause(err),
+                        isTimeout: isTimeoutAbortError(err),
+                    }, "upsertPointsOneTable: BulkUpsert aborted");
+                }
+                throw err;
+            }
+        }, {
+            isTransient: isTransientYdbError,
+            context: {
+                operation: "upsertPointsOneTable",
+                tableName,
+                uid,
+                batchStart: i,
+                batchSize: batch.length,
+            },
+        });
+        upserted += batch.length;
+    }
+    return upserted;
+}

package/dist/repositories/pointsRepo.one-table.d.ts

@@ -1,13 +1,3 @@
-import type { DistanceKind } from "../types";
-import { SearchMode } from "../config/env.js";
-export declare function upsertPointsOneTable(tableName: string, points: Array<{
-    id: string | number;
-    vector: number[];
-    payload?: Record<string, unknown>;
-}>, dimension: number, uid: string): Promise<number>;
-export declare function searchPointsOneTable(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number, uid: string, mode: SearchMode | undefined, overfetchMultiplier: number): Promise<Array<{
-    id: string;
-    score: number;
-    payload?: Record<string, unknown>;
-}>>;
-export declare function deletePointsOneTable(tableName: string, ids: Array<string | number>, uid: string): Promise<number>;
+export { searchPointsOneTable } from "./pointsRepo.one-table/Search/index.js";
+export { upsertPointsOneTable } from "./pointsRepo.one-table/Upsert.js";
+export { deletePointsOneTable, deletePointsByPathSegmentsOneTable, } from "./pointsRepo.one-table/Delete.js";