ydb-qdrant 5.1.0 → 5.2.0

package/dist/config/env.d.ts

@@ -4,6 +4,7 @@ export declare const YDB_DATABASE: string;
 export declare const PORT: number;
 export declare const LOG_LEVEL: string;
 export declare const GLOBAL_POINTS_AUTOMIGRATE_ENABLED: boolean;
+export declare const USE_BATCH_DELETE_FOR_COLLECTIONS: boolean;
 export declare enum SearchMode {
     Exact = "exact",
     Approximate = "approximate"
@@ -16,5 +17,6 @@ export declare const UPSERT_BATCH_SIZE: number;
 export declare const SESSION_POOL_MIN_SIZE: number;
 export declare const SESSION_POOL_MAX_SIZE: number;
 export declare const SESSION_KEEPALIVE_PERIOD_MS: number;
+export declare const STARTUP_PROBE_SESSION_TIMEOUT_MS: number;
 export declare const UPSERT_OPERATION_TIMEOUT_MS: number;
 export declare const SEARCH_OPERATION_TIMEOUT_MS: number;

package/dist/config/env.js

@@ -35,6 +35,7 @@ function parseBooleanEnv(value, defaultValue) {
     return true;
 }
 export const GLOBAL_POINTS_AUTOMIGRATE_ENABLED = parseBooleanEnv(process.env.YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE, false);
+export const USE_BATCH_DELETE_FOR_COLLECTIONS = parseBooleanEnv(process.env.YDB_QDRANT_USE_BATCH_DELETE, true);
 export var SearchMode;
 (function (SearchMode) {
     SearchMode["Exact"] = "exact";
@@ -67,5 +68,6 @@ const NORMALIZED_SESSION_POOL_MIN_SIZE = RAW_SESSION_POOL_MIN_SIZE > RAW_SESSION
 export const SESSION_POOL_MIN_SIZE = NORMALIZED_SESSION_POOL_MIN_SIZE;
 export const SESSION_POOL_MAX_SIZE = RAW_SESSION_POOL_MAX_SIZE;
 export const SESSION_KEEPALIVE_PERIOD_MS = parseIntegerEnv(process.env.YDB_SESSION_KEEPALIVE_PERIOD_MS, 5000, { min: 1000, max: 60000 });
+export const STARTUP_PROBE_SESSION_TIMEOUT_MS = parseIntegerEnv(process.env.YDB_QDRANT_STARTUP_PROBE_SESSION_TIMEOUT_MS, 5000, { min: 1000 });
 export const UPSERT_OPERATION_TIMEOUT_MS = parseIntegerEnv(process.env.YDB_QDRANT_UPSERT_TIMEOUT_MS, 5000, { min: 1000 });
 export const SEARCH_OPERATION_TIMEOUT_MS = parseIntegerEnv(process.env.YDB_QDRANT_SEARCH_TIMEOUT_MS, 10000, { min: 1000 });

package/dist/repositories/collectionsRepo.js

@@ -1,4 +1,5 @@
 import { TypedValues, withSession, createExecuteQuerySettings, withStartupProbeSession, createExecuteQuerySettingsWithTimeout, } from "../ydb/client.js";
+import { STARTUP_PROBE_SESSION_TIMEOUT_MS } from "../config/env.js";
 import { uidFor } from "../utils/tenant.js";
 import { createCollectionOneTable, deleteCollectionOneTable, } from "./collectionsRepo.one-table.js";
 import { withRetry, isTransientYdbError } from "../utils/retry.js";
@@ -41,7 +42,7 @@ export async function verifyCollectionsQueryCompilationForStartup() {
             const settings = createExecuteQuerySettingsWithTimeout({
                 keepInCache: true,
                 idempotent: true,
-                timeoutMs: 3000,
+                timeoutMs: STARTUP_PROBE_SESSION_TIMEOUT_MS,
             });
             await s.executeQuery(qry, {
                 $collection: TypedValues.utf8(probeKey),

package/dist/repositories/collectionsRepo.one-table.js

@@ -2,6 +2,7 @@ import { TypedValues, Types, withSession, createExecuteQuerySettings, } from "..
 import { GLOBAL_POINTS_TABLE, ensureGlobalPointsTable } from "../ydb/schema.js";
 import { upsertCollectionMeta } from "./collectionsRepo.shared.js";
 import { withRetry, isTransientYdbError } from "../utils/retry.js";
+import { USE_BATCH_DELETE_FOR_COLLECTIONS } from "../config/env.js";
 const DELETE_COLLECTION_BATCH_SIZE = 10000;
 function isOutOfBufferMemoryYdbError(error) {
     const msg = error instanceof Error ? error.message : String(error);
@@ -63,32 +64,68 @@ export async function createCollectionOneTable(metaKey, dim, distance, vectorTyp
 }
 export async function deleteCollectionOneTable(metaKey, uid) {
     await ensureGlobalPointsTable();
-    const deletePointsYql = `
-    DECLARE $uid AS Utf8;
-    DELETE FROM ${GLOBAL_POINTS_TABLE} WHERE uid = $uid;
-  `;
-    await withRetry(() => withSession(async (s) => {
-        const settings = createExecuteQuerySettings();
-        try {
-            await s.executeQuery(deletePointsYql, {
-                $uid: TypedValues.utf8(uid),
-            }, undefined, settings);
-        }
-        catch (err) {
-            if (!isOutOfBufferMemoryYdbError(err)) {
-                throw err;
+    if (USE_BATCH_DELETE_FOR_COLLECTIONS) {
+        const batchDeletePointsYql = `
+      DECLARE $uid AS Utf8;
+      BATCH DELETE FROM ${GLOBAL_POINTS_TABLE}
+      WHERE uid = $uid;
+    `;
+        await withRetry(() => withSession(async (s) => {
+            const settings = createExecuteQuerySettings();
+            try {
+                await s.executeQuery(batchDeletePointsYql, {
+                    $uid: TypedValues.utf8(uid),
+                }, undefined, settings);
             }
-            await deletePointsForUidInChunks(s, uid);
-        }
-    }), {
-        isTransient: isTransientYdbError,
-        context: {
-            operation: "deleteCollectionOneTable",
-            tableName: GLOBAL_POINTS_TABLE,
-            metaKey,
-            uid,
-        },
-    });
+            catch (err) {
+                if (!isOutOfBufferMemoryYdbError(err)) {
+                    throw err;
+                }
+                // BATCH DELETE already deletes in chunks per partition, but if YDB
+                // still reports an out-of-buffer-memory condition, fall back to
+                // the same per-uid chunked deletion strategy as the legacy path.
+                await deletePointsForUidInChunks(s, uid);
+            }
+        }), {
+            isTransient: isTransientYdbError,
+            context: {
+                operation: "deleteCollectionOneTable",
+                tableName: GLOBAL_POINTS_TABLE,
+                metaKey,
+                uid,
+                mode: "batch_delete",
+            },
+        });
+    }
+    else {
+        const deletePointsYql = `
+      DECLARE $uid AS Utf8;
+      DELETE FROM ${GLOBAL_POINTS_TABLE} WHERE uid = $uid;
+    `;
+        await withRetry(() => withSession(async (s) => {
+            const settings = createExecuteQuerySettings();
+            try {
+                await s.executeQuery(deletePointsYql, {
+                    $uid: TypedValues.utf8(uid),
+                }, undefined, settings);
+            }
+            catch (err) {
+                if (!isOutOfBufferMemoryYdbError(err)) {
+                    throw err;
+                }
+                await deletePointsForUidInChunks(s, uid);
+            }
+        }), {
+            isTransient: isTransientYdbError,
+            context: {
+                operation: "deleteCollectionOneTable",
+                tableName: GLOBAL_POINTS_TABLE,
+                metaKey,
+                uid,
+                mode: "legacy_chunked",
+            },
+        });
+    }
     const delMeta = `
     DECLARE $collection AS Utf8;
     DELETE FROM qdr__collections WHERE collection = $collection;

package/dist/utils/retry.js

@@ -32,7 +32,14 @@ export async function withRetry(fn, options = {}) {
                 throw e;
             }
             const backoffMs = Math.floor(baseDelayMs * Math.pow(2, attempt) + Math.random() * 100);
-            logger.warn({ ...context, attempt, backoffMs }, "operation aborted due to transient error; retrying");
+            logger.warn({
+                ...context,
+                attempt,
+                backoffMs,
+                err: e instanceof Error
+                    ? e
+                    : new Error(typeof e === "string" ? e : JSON.stringify(e)),
+            }, "operation aborted due to transient error; retrying");
             await new Promise((r) => setTimeout(r, backoffMs));
             attempt += 1;
         }

package/dist/ydb/client.js

@@ -1,5 +1,5 @@
 import { createRequire } from "module";
-import { YDB_DATABASE, YDB_ENDPOINT, SESSION_POOL_MIN_SIZE, SESSION_POOL_MAX_SIZE, SESSION_KEEPALIVE_PERIOD_MS, } from "../config/env.js";
+import { YDB_DATABASE, YDB_ENDPOINT, SESSION_POOL_MIN_SIZE, SESSION_POOL_MAX_SIZE, SESSION_KEEPALIVE_PERIOD_MS, STARTUP_PROBE_SESSION_TIMEOUT_MS, } from "../config/env.js";
 import { logger } from "../logging/logger.js";
 const require = createRequire(import.meta.url);
 const { Driver, getCredentialsFromEnv, Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, OperationParams, Ydb, } = require("ydb-sdk");
@@ -30,7 +30,6 @@ const DRIVER_READY_TIMEOUT_MS = 15000;
 const TABLE_SESSION_TIMEOUT_MS = 20000;
 const YDB_HEALTHCHECK_READY_TIMEOUT_MS = 5000;
 const DRIVER_REFRESH_COOLDOWN_MS = 30000;
-const STARTUP_PROBE_SESSION_TIMEOUT_MS = 3000;
 let overrideConfig;
 let driver;
 let lastDriverRefreshAt = 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ydb-qdrant",
-  "version": "5.1.0",
+  "version": "5.2.0",
   "main": "dist/package/api.js",
   "types": "dist/package/api.d.ts",
   "exports": {
@@ -50,6 +50,14 @@
   "author": "",
   "license": "Apache-2.0",
   "description": "Qdrant-compatible Node.js/TypeScript API that stores/searches embeddings in YDB using a global one-table layout with exact and approximate KNN search over serialized vectors.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/astandrik/ydb-qdrant.git"
+  },
+  "homepage": "https://github.com/astandrik/ydb-qdrant#readme",
+  "bugs": {
+    "url": "https://github.com/astandrik/ydb-qdrant/issues"
+  },
   "type": "module",
   "publishConfig": {
     "access": "public"
@@ -83,4 +91,4 @@
     "typescript-eslint": "^8.47.0",
     "vitest": "^4.0.12"
   }
-}
+}