ydb-qdrant 4.8.0 → 4.8.1

package/dist/repositories/collectionsRepo.js

@@ -1,4 +1,4 @@
-import { TypedValues, withSession } from "../ydb/client.js";
+import { TypedValues, withSession, createExecuteQuerySettings, } from "../ydb/client.js";
 import { mapDistanceToIndexParam } from "../utils/distance.js";
 import { COLLECTION_STORAGE_MODE, isOneTableMode, } from "../config/env.js";
 import { GLOBAL_POINTS_TABLE } from "../ydb/schema.js";
@@ -20,9 +20,10 @@ export async function getCollectionMeta(metaKey) {
     WHERE collection = $collection;
   `;
     const res = await withSession(async (s) => {
+        const settings = createExecuteQuerySettings();
         return await s.executeQuery(qry, {
             $collection: TypedValues.utf8(metaKey),
-        });
+        }, undefined, settings);
     });
     const rowset = res.resultSets?.[0];
     if (!rowset || rowset.rows?.length !== 1)

package/dist/repositories/collectionsRepo.multi-table.js

@@ -1,4 +1,4 @@
-import { Types, TypedValues, withSession, TableDescription, Column, } from "../ydb/client.js";
+import { Types, TypedValues, withSession, TableDescription, Column, createExecuteQuerySettings, } from "../ydb/client.js";
 import { upsertCollectionMeta } from "./collectionsRepo.shared.js";
 export async function createCollectionMultiTable(metaKey, dim, distance, vectorType, tableName) {
     await withSession(async (s) => {
@@ -18,6 +18,7 @@ export async function deleteCollectionMultiTable(metaKey, tableName) {
     DELETE FROM qdr__collections WHERE collection = $collection;
   `;
     await withSession(async (s) => {
-        await s.executeQuery(delMeta, { $collection: TypedValues.utf8(metaKey) });
+        const settings = createExecuteQuerySettings();
+        await s.executeQuery(delMeta, { $collection: TypedValues.utf8(metaKey) }, undefined, settings);
     });
 }

package/dist/repositories/collectionsRepo.one-table.js

@@ -1,6 +1,63 @@
-import { TypedValues, withSession } from "../ydb/client.js";
+import { TypedValues, Types, withSession, createExecuteQuerySettings, } from "../ydb/client.js";
 import { GLOBAL_POINTS_TABLE, ensureGlobalPointsTable } from "../ydb/schema.js";
 import { upsertCollectionMeta } from "./collectionsRepo.shared.js";
+import { withRetry, isTransientYdbError } from "../utils/retry.js";
+const DELETE_COLLECTION_BATCH_SIZE = 10000;
+function isOutOfBufferMemoryYdbError(error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    if (/Out of buffer memory/i.test(msg)) {
+        return true;
+    }
+    if (typeof error === "object" && error !== null) {
+        const issues = error.issues;
+        if (issues !== undefined) {
+            const issuesText = typeof issues === "string" ? issues : JSON.stringify(issues);
+            return /Out of buffer memory/i.test(issuesText);
+        }
+    }
+    return false;
+}
+async function deletePointsForUidInChunks(s, uid) {
+    const selectYql = `
+    DECLARE $uid AS Utf8;
+    DECLARE $limit AS Uint32;
+    SELECT point_id
+    FROM ${GLOBAL_POINTS_TABLE}
+    WHERE uid = $uid
+    LIMIT $limit;
+  `;
+    const deleteBatchYql = `
+    DECLARE $uid AS Utf8;
+    DECLARE $ids AS List<Utf8>;
+    DELETE FROM ${GLOBAL_POINTS_TABLE}
+    WHERE uid = $uid AND point_id IN $ids;
+  `;
+    // Best‑effort loop: stop when there are no more rows for this uid.
+    // Each iteration only touches a limited number of rows to avoid
+    // hitting YDB's per‑operation buffer limits.
+    let iterations = 0;
+    const MAX_ITERATIONS = 1000;
+    const settings = createExecuteQuerySettings();
+    while (iterations++ < MAX_ITERATIONS) {
+        const rs = (await s.executeQuery(selectYql, {
+            $uid: TypedValues.utf8(uid),
+            $limit: TypedValues.uint32(DELETE_COLLECTION_BATCH_SIZE),
+        }, undefined, settings));
+        const rowset = rs.resultSets?.[0];
+        const rows = rowset?.rows ?? [];
+        const ids = rows
+            .map((row) => row.items?.[0]?.textValue)
+            .filter((id) => typeof id === "string");
+        if (ids.length === 0) {
+            break;
+        }
+        const idsValue = TypedValues.list(Types.UTF8, ids);
+        await s.executeQuery(deleteBatchYql, {
+            $uid: TypedValues.utf8(uid),
+            $ids: idsValue,
+        }, undefined, settings);
+    }
+}
 export async function createCollectionOneTable(metaKey, dim, distance, vectorType) {
     await upsertCollectionMeta(metaKey, dim, distance, vectorType, GLOBAL_POINTS_TABLE);
 }
@@ -10,16 +67,36 @@ export async function deleteCollectionOneTable(metaKey, uid) {
     DECLARE $uid AS Utf8;
     DELETE FROM ${GLOBAL_POINTS_TABLE} WHERE uid = $uid;
   `;
-    await withSession(async (s) => {
-        await s.executeQuery(deletePointsYql, {
-            $uid: TypedValues.utf8(uid),
-        });
+    await withRetry(() => withSession(async (s) => {
+        const settings = createExecuteQuerySettings();
+        try {
+            await s.executeQuery(deletePointsYql, {
+                $uid: TypedValues.utf8(uid),
+            }, undefined, settings);
+        }
+        catch (err) {
+            if (!isOutOfBufferMemoryYdbError(err)) {
+                throw err;
+            }
+            await deletePointsForUidInChunks(s, uid);
+        }
+    }), {
+        isTransient: isTransientYdbError,
+        context: {
+            operation: "deleteCollectionOneTable",
+            tableName: GLOBAL_POINTS_TABLE,
+            metaKey,
+            uid,
+        },
     });
     const delMeta = `
     DECLARE $collection AS Utf8;
     DELETE FROM qdr__collections WHERE collection = $collection;
   `;
     await withSession(async (s) => {
-        await s.executeQuery(delMeta, { $collection: TypedValues.utf8(metaKey) });
+        const settings = createExecuteQuerySettings();
+        await s.executeQuery(delMeta, {
+            $collection: TypedValues.utf8(metaKey),
+        }, undefined, settings);
     });
 }

package/dist/repositories/pointsRepo.multi-table.js

@@ -1,4 +1,4 @@
-import { Types, TypedValues, withSession } from "../ydb/client.js";
+import { Types, TypedValues, withSession, createExecuteQuerySettings, } from "../ydb/client.js";
 import { buildVectorParam } from "../ydb/helpers.js";
 import { logger } from "../logging/logger.js";
 import { notifyUpsert } from "../indexing/IndexScheduler.js";
@@ -15,6 +15,7 @@ export async function upsertPointsMultiTable(tableName, points, dimension) {
     }
     let upserted = 0;
     await withSession(async (s) => {
+        const settings = createExecuteQuerySettings();
         for (let i = 0; i < points.length; i += UPSERT_BATCH_SIZE) {
             const batch = points.slice(i, i + UPSERT_BATCH_SIZE);
             const ddl = `
@@ -44,7 +45,7 @@ export async function upsertPointsMultiTable(tableName, points, dimension) {
             const params = {
                 $rows: rowsValue,
             };
-            await withRetry(() => s.executeQuery(ddl, params), {
+            await withRetry(() => s.executeQuery(ddl, params, undefined, settings), {
                 isTransient: isTransientYdbError,
                 context: { tableName, batchSize: batch.length },
             });
@@ -64,6 +65,7 @@ export async function searchPointsMultiTable(tableName, queryVector, top, withPa
         $qf: qf,
         $k2: TypedValues.uint32(top),
     };
+    const settings = createExecuteQuerySettings();
     const buildQuery = (useIndex) => `
     DECLARE $qf AS List<Float>;
     DECLARE $k2 AS Uint32;
@@ -77,7 +79,7 @@ export async function searchPointsMultiTable(tableName, queryVector, top, withPa
     if (VECTOR_INDEX_BUILD_ENABLED) {
         try {
             rs = await withSession(async (s) => {
-                return await s.executeQuery(buildQuery(true), params);
+                return await s.executeQuery(buildQuery(true), params, undefined, settings);
             });
             logger.info({ tableName }, "vector index found; using index for search");
         }
@@ -87,7 +89,7 @@ export async function searchPointsMultiTable(tableName, queryVector, top, withPa
             if (indexUnavailable) {
                 logger.info({ tableName }, "vector index not available (missing or building); falling back to table scan");
                 rs = await withSession(async (s) => {
-                    return await s.executeQuery(buildQuery(false), params);
+                    return await s.executeQuery(buildQuery(false), params, undefined, settings);
                 });
             }
             else {
@@ -97,7 +99,7 @@ export async function searchPointsMultiTable(tableName, queryVector, top, withPa
     }
     else {
         rs = await withSession(async (s) => {
-            return await s.executeQuery(buildQuery(false), params);
+            return await s.executeQuery(buildQuery(false), params, undefined, settings);
         });
     }
     const rowset = rs.resultSets?.[0];
@@ -128,6 +130,7 @@ export async function searchPointsMultiTable(tableName, queryVector, top, withPa
 export async function deletePointsMultiTable(tableName, ids) {
     let deleted = 0;
     await withSession(async (s) => {
+        const settings = createExecuteQuerySettings();
         for (const id of ids) {
             const yql = `
         DECLARE $id AS Utf8;
@@ -136,7 +139,7 @@ export async function deletePointsMultiTable(tableName, ids) {
             const params = {
                 $id: TypedValues.utf8(String(id)),
             };
-            await s.executeQuery(yql, params);
+            await s.executeQuery(yql, params, undefined, settings);
             deleted += 1;
         }
     });

package/dist/repositories/pointsRepo.one-table.js

@@ -1,4 +1,4 @@
-import { TypedValues, Types, withSession } from "../ydb/client.js";
+import { TypedValues, Types, withSession, createExecuteQuerySettings, } from "../ydb/client.js";
 import { buildVectorParam, buildVectorBinaryParams } from "../ydb/helpers.js";
 import { notifyUpsert } from "../indexing/IndexScheduler.js";
 import { mapDistanceToKnnFn, mapDistanceToBitKnnFn, } from "../utils/distance.js";
@@ -25,6 +25,7 @@ export async function upsertPointsOneTable(tableName, points, dimension, uid) {
     }
     let upserted = 0;
     await withSession(async (s) => {
+        const settings = createExecuteQuerySettings();
         for (let i = 0; i < points.length; i += UPSERT_BATCH_SIZE) {
             const batch = points.slice(i, i + UPSERT_BATCH_SIZE);
             let ddl;
@@ -120,7 +121,7 @@ export async function upsertPointsOneTable(tableName, points, dimension, uid) {
                     })),
                 },
             }, "one_table upsert: executing YQL");
-            await withRetry(() => s.executeQuery(ddl, params), {
+            await withRetry(() => s.executeQuery(ddl, params, undefined, settings), {
                 isTransient: isTransientYdbError,
                 context: { tableName, batchSize: batch.length },
             });
@@ -193,7 +194,8 @@ async function searchPointsOneTableExact(tableName, queryVector, top, withPayloa
                 vectorPreview: queryVector.slice(0, 3),
             },
         }, "one_table search (exact): executing YQL");
-        const rs = await s.executeQuery(yql, params);
+        const settings = createExecuteQuerySettings();
+        const rs = await s.executeQuery(yql, params, undefined, settings);
         const rowset = rs.resultSets?.[0];
         const rows = (rowset?.rows ?? []);
         return rows.map((row) => {
@@ -334,7 +336,8 @@ async function searchPointsOneTableApproximate(tableName, queryVector, top, with
                 },
             }, "one_table search (approximate): executing YQL");
         }
-        const rs = await s.executeQuery(yql, params);
+        const settings = createExecuteQuerySettings();
+        const rs = await s.executeQuery(yql, params, undefined, settings);
         const rowset = rs.resultSets?.[0];
         const rows = (rowset?.rows ?? []);
         return rows.map((row) => {
@@ -371,6 +374,7 @@ export async function searchPointsOneTable(tableName, queryVector, top, withPayl
 export async function deletePointsOneTable(tableName, ids, uid) {
     let deleted = 0;
     await withSession(async (s) => {
+        const settings = createExecuteQuerySettings();
         for (const id of ids) {
             const yql = `
         DECLARE $uid AS Utf8;
@@ -381,7 +385,7 @@ export async function deletePointsOneTable(tableName, ids, uid) {
                 $uid: TypedValues.utf8(uid),
                 $id: TypedValues.utf8(String(id)),
             };
-            await s.executeQuery(yql, params);
+            await s.executeQuery(yql, params, undefined, settings);
             deleted += 1;
         }
     });

package/dist/utils/retry.js

@@ -3,7 +3,19 @@ const DEFAULT_MAX_RETRIES = 6;
 const DEFAULT_BASE_DELAY_MS = 250;
 export function isTransientYdbError(error) {
     const msg = error instanceof Error ? error.message : String(error);
-    return /Aborted|schema version mismatch|Table metadata loading|Failed to load metadata/i.test(msg);
+    if (/Aborted|schema version mismatch|Table metadata loading|Failed to load metadata|overloaded|is in process of split|wrong shard state|Rejecting data TxId/i.test(msg)) {
+        return true;
+    }
+    if (typeof error === "object" && error !== null) {
+        const issues = error.issues;
+        if (issues !== undefined) {
+            const issuesText = typeof issues === "string" ? issues : JSON.stringify(issues);
+            if (/overloaded|is in process of split|wrong shard state|Rejecting data TxId/i.test(issuesText)) {
+                return true;
+            }
+        }
+    }
+    return false;
 }
 export async function withRetry(fn, options = {}) {
     const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;

package/dist/ydb/client.d.ts

@@ -1,6 +1,10 @@
-import type { Session, IAuthService } from "ydb-sdk";
-declare const Types: typeof import("ydb-sdk").Types, TypedValues: typeof import("ydb-sdk").TypedValues, TableDescription: typeof import("ydb-sdk").TableDescription, Column: typeof import("ydb-sdk").Column;
-export { Types, TypedValues, TableDescription, Column };
+import type { Session, IAuthService, ExecuteQuerySettings as YdbExecuteQuerySettings } from "ydb-sdk";
+declare const Types: typeof import("ydb-sdk").Types, TypedValues: typeof import("ydb-sdk").TypedValues, TableDescription: typeof import("ydb-sdk").TableDescription, Column: typeof import("ydb-sdk").Column, ExecuteQuerySettings: typeof YdbExecuteQuerySettings;
+export { Types, TypedValues, TableDescription, Column, ExecuteQuerySettings };
+export declare function createExecuteQuerySettings(options?: {
+    keepInCache?: boolean;
+    idempotent?: boolean;
+}): YdbExecuteQuerySettings;
 type DriverConfig = {
     endpoint?: string;
     database?: string;

package/dist/ydb/client.js

@@ -2,8 +2,19 @@ import { createRequire } from "module";
 import { YDB_DATABASE, YDB_ENDPOINT, SESSION_POOL_MIN_SIZE, SESSION_POOL_MAX_SIZE, SESSION_KEEPALIVE_PERIOD_MS, } from "../config/env.js";
 import { logger } from "../logging/logger.js";
 const require = createRequire(import.meta.url);
-const { Driver, getCredentialsFromEnv, Types, TypedValues, TableDescription, Column, } = require("ydb-sdk");
-export { Types, TypedValues, TableDescription, Column };
+const { Driver, getCredentialsFromEnv, Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, } = require("ydb-sdk");
+export { Types, TypedValues, TableDescription, Column, ExecuteQuerySettings };
+export function createExecuteQuerySettings(options) {
+    const { keepInCache = true, idempotent = true } = options ?? {};
+    const settings = new ExecuteQuerySettings();
+    if (keepInCache) {
+        settings.withKeepInCache(true);
+    }
+    if (idempotent) {
+        settings.withIdempotent(true);
+    }
+    return settings;
+}
 const DRIVER_READY_TIMEOUT_MS = 15000;
 const TABLE_SESSION_TIMEOUT_MS = 20000;
 const YDB_HEALTHCHECK_READY_TIMEOUT_MS = 5000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ydb-qdrant",
-  "version": "4.8.0",
+  "version": "4.8.1",
   "main": "dist/package/api.js",
   "types": "dist/package/api.d.ts",
   "exports": {