ydb-qdrant 5.0.0 → 5.2.0

package/dist/config/env.d.ts

@@ -4,6 +4,7 @@ export declare const YDB_DATABASE: string;
 export declare const PORT: number;
 export declare const LOG_LEVEL: string;
 export declare const GLOBAL_POINTS_AUTOMIGRATE_ENABLED: boolean;
+export declare const USE_BATCH_DELETE_FOR_COLLECTIONS: boolean;
 export declare enum SearchMode {
     Exact = "exact",
     Approximate = "approximate"
@@ -16,3 +17,6 @@ export declare const UPSERT_BATCH_SIZE: number;
 export declare const SESSION_POOL_MIN_SIZE: number;
 export declare const SESSION_POOL_MAX_SIZE: number;
 export declare const SESSION_KEEPALIVE_PERIOD_MS: number;
+export declare const STARTUP_PROBE_SESSION_TIMEOUT_MS: number;
+export declare const UPSERT_OPERATION_TIMEOUT_MS: number;
+export declare const SEARCH_OPERATION_TIMEOUT_MS: number;

package/dist/config/env.js

@@ -35,6 +35,7 @@ function parseBooleanEnv(value, defaultValue) {
     return true;
 }
 export const GLOBAL_POINTS_AUTOMIGRATE_ENABLED = parseBooleanEnv(process.env.YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE, false);
+export const USE_BATCH_DELETE_FOR_COLLECTIONS = parseBooleanEnv(process.env.YDB_QDRANT_USE_BATCH_DELETE, true);
 export var SearchMode;
 (function (SearchMode) {
     SearchMode["Exact"] = "exact";
@@ -67,3 +68,6 @@ const NORMALIZED_SESSION_POOL_MIN_SIZE = RAW_SESSION_POOL_MIN_SIZE > RAW_SESSION
 export const SESSION_POOL_MIN_SIZE = NORMALIZED_SESSION_POOL_MIN_SIZE;
 export const SESSION_POOL_MAX_SIZE = RAW_SESSION_POOL_MAX_SIZE;
 export const SESSION_KEEPALIVE_PERIOD_MS = parseIntegerEnv(process.env.YDB_SESSION_KEEPALIVE_PERIOD_MS, 5000, { min: 1000, max: 60000 });
+export const STARTUP_PROBE_SESSION_TIMEOUT_MS = parseIntegerEnv(process.env.YDB_QDRANT_STARTUP_PROBE_SESSION_TIMEOUT_MS, 5000, { min: 1000 });
+export const UPSERT_OPERATION_TIMEOUT_MS = parseIntegerEnv(process.env.YDB_QDRANT_UPSERT_TIMEOUT_MS, 5000, { min: 1000 });
+export const SEARCH_OPERATION_TIMEOUT_MS = parseIntegerEnv(process.env.YDB_QDRANT_SEARCH_TIMEOUT_MS, 10000, { min: 1000 });

package/dist/repositories/collectionsRepo.js

@@ -1,6 +1,8 @@
 import { TypedValues, withSession, createExecuteQuerySettings, withStartupProbeSession, createExecuteQuerySettingsWithTimeout, } from "../ydb/client.js";
+import { STARTUP_PROBE_SESSION_TIMEOUT_MS } from "../config/env.js";
 import { uidFor } from "../utils/tenant.js";
 import { createCollectionOneTable, deleteCollectionOneTable, } from "./collectionsRepo.one-table.js";
+import { withRetry, isTransientYdbError } from "../utils/retry.js";
 export async function createCollection(metaKey, dim, distance, vectorType) {
     await createCollectionOneTable(metaKey, dim, distance, vectorType);
 }
@@ -35,15 +37,22 @@ export async function verifyCollectionsQueryCompilationForStartup() {
     FROM qdr__collections
     WHERE collection = $collection;
   `;
-    await withStartupProbeSession(async (s) => {
-        const settings = createExecuteQuerySettingsWithTimeout({
-            keepInCache: true,
-            idempotent: true,
-            timeoutMs: 3000,
+    await withRetry(async () => {
+        await withStartupProbeSession(async (s) => {
+            const settings = createExecuteQuerySettingsWithTimeout({
+                keepInCache: true,
+                idempotent: true,
+                timeoutMs: STARTUP_PROBE_SESSION_TIMEOUT_MS,
+            });
+            await s.executeQuery(qry, {
+                $collection: TypedValues.utf8(probeKey),
+            }, undefined, settings);
         });
-        await s.executeQuery(qry, {
-            $collection: TypedValues.utf8(probeKey),
-        }, undefined, settings);
+    }, {
+        isTransient: isTransientYdbError,
+        maxRetries: 2,
+        baseDelayMs: 200,
+        context: { probe: "collections_startup_compilation" },
     });
 }
 export async function deleteCollection(metaKey, uid) {

package/dist/repositories/collectionsRepo.one-table.js

@@ -2,6 +2,7 @@ import { TypedValues, Types, withSession, createExecuteQuerySettings, } from "..
 import { GLOBAL_POINTS_TABLE, ensureGlobalPointsTable } from "../ydb/schema.js";
 import { upsertCollectionMeta } from "./collectionsRepo.shared.js";
 import { withRetry, isTransientYdbError } from "../utils/retry.js";
+import { USE_BATCH_DELETE_FOR_COLLECTIONS } from "../config/env.js";
 const DELETE_COLLECTION_BATCH_SIZE = 10000;
 function isOutOfBufferMemoryYdbError(error) {
     const msg = error instanceof Error ? error.message : String(error);
@@ -63,32 +64,68 @@ export async function createCollectionOneTable(metaKey, dim, distance, vectorTyp
 }
 export async function deleteCollectionOneTable(metaKey, uid) {
     await ensureGlobalPointsTable();
-    const deletePointsYql = `
-    DECLARE $uid AS Utf8;
-    DELETE FROM ${GLOBAL_POINTS_TABLE} WHERE uid = $uid;
-  `;
-    await withRetry(() => withSession(async (s) => {
-        const settings = createExecuteQuerySettings();
-        try {
-            await s.executeQuery(deletePointsYql, {
-                $uid: TypedValues.utf8(uid),
-            }, undefined, settings);
-        }
-        catch (err) {
-            if (!isOutOfBufferMemoryYdbError(err)) {
-                throw err;
+    if (USE_BATCH_DELETE_FOR_COLLECTIONS) {
+        const batchDeletePointsYql = `
+      DECLARE $uid AS Utf8;
+      BATCH DELETE FROM ${GLOBAL_POINTS_TABLE}
+      WHERE uid = $uid;
+    `;
+        await withRetry(() => withSession(async (s) => {
+            const settings = createExecuteQuerySettings();
+            try {
+                await s.executeQuery(batchDeletePointsYql, {
+                    $uid: TypedValues.utf8(uid),
+                }, undefined, settings);
             }
-            await deletePointsForUidInChunks(s, uid);
-        }
-    }), {
-        isTransient: isTransientYdbError,
-        context: {
-            operation: "deleteCollectionOneTable",
-            tableName: GLOBAL_POINTS_TABLE,
-            metaKey,
-            uid,
-        },
-    });
+            catch (err) {
+                if (!isOutOfBufferMemoryYdbError(err)) {
+                    throw err;
+                }
+                // BATCH DELETE already deletes in chunks per partition, but if YDB
+                // still reports an out-of-buffer-memory condition, fall back to
+                // the same per-uid chunked deletion strategy as the legacy path.
+                await deletePointsForUidInChunks(s, uid);
+            }
+        }), {
+            isTransient: isTransientYdbError,
+            context: {
+                operation: "deleteCollectionOneTable",
+                tableName: GLOBAL_POINTS_TABLE,
+                metaKey,
+                uid,
+                mode: "batch_delete",
+            },
+        });
+    }
+    else {
+        const deletePointsYql = `
+      DECLARE $uid AS Utf8;
+      DELETE FROM ${GLOBAL_POINTS_TABLE} WHERE uid = $uid;
+    `;
+        await withRetry(() => withSession(async (s) => {
+            const settings = createExecuteQuerySettings();
+            try {
+                await s.executeQuery(deletePointsYql, {
+                    $uid: TypedValues.utf8(uid),
+                }, undefined, settings);
+            }
+            catch (err) {
+                if (!isOutOfBufferMemoryYdbError(err)) {
+                    throw err;
+                }
+                await deletePointsForUidInChunks(s, uid);
+            }
+        }), {
+            isTransient: isTransientYdbError,
+            context: {
+                operation: "deleteCollectionOneTable",
+                tableName: GLOBAL_POINTS_TABLE,
+                metaKey,
+                uid,
+                mode: "legacy_chunked",
+            },
+        });
+    }
     const delMeta = `
     DECLARE $collection AS Utf8;
     DELETE FROM qdr__collections WHERE collection = $collection;

package/dist/repositories/pointsRepo.one-table.js

@@ -1,9 +1,9 @@
-import { TypedValues, Types, withSession, createExecuteQuerySettings, } from "../ydb/client.js";
+import { TypedValues, Types, withSession, createExecuteQuerySettings, createExecuteQuerySettingsWithTimeout, } from "../ydb/client.js";
 import { buildVectorParam, buildVectorBinaryParams } from "../ydb/helpers.js";
 import { mapDistanceToKnnFn, mapDistanceToBitKnnFn, } from "../utils/distance.js";
 import { withRetry, isTransientYdbError } from "../utils/retry.js";
 import { UPSERT_BATCH_SIZE } from "../ydb/schema.js";
-import { CLIENT_SIDE_SERIALIZATION_ENABLED, SearchMode, } from "../config/env.js";
+import { CLIENT_SIDE_SERIALIZATION_ENABLED, SearchMode, UPSERT_OPERATION_TIMEOUT_MS, SEARCH_OPERATION_TIMEOUT_MS, } from "../config/env.js";
 import { logger } from "../logging/logger.js";
 export async function upsertPointsOneTable(tableName, points, dimension, uid) {
     for (const p of points) {
@@ -24,7 +24,11 @@ export async function upsertPointsOneTable(tableName, points, dimension, uid) {
     }
     let upserted = 0;
     await withSession(async (s) => {
-        const settings = createExecuteQuerySettings();
+        const settings = createExecuteQuerySettingsWithTimeout({
+            keepInCache: true,
+            idempotent: true,
+            timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
+        });
         for (let i = 0; i < points.length; i += UPSERT_BATCH_SIZE) {
             const batch = points.slice(i, i + UPSERT_BATCH_SIZE);
             let ddl;
@@ -192,7 +196,11 @@ async function searchPointsOneTableExact(tableName, queryVector, top, withPayloa
                 vectorPreview: queryVector.slice(0, 3),
             },
         }, "one_table search (exact): executing YQL");
-        const settings = createExecuteQuerySettings();
+        const settings = createExecuteQuerySettingsWithTimeout({
+            keepInCache: true,
+            idempotent: true,
+            timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
+        });
         const rs = await s.executeQuery(yql, params, undefined, settings);
         const rowset = rs.resultSets?.[0];
         const rows = (rowset?.rows ?? []);
@@ -334,7 +342,11 @@ async function searchPointsOneTableApproximate(tableName, queryVector, top, with
                 },
             }, "one_table search (approximate): executing YQL");
         }
-        const settings = createExecuteQuerySettings();
+        const settings = createExecuteQuerySettingsWithTimeout({
+            keepInCache: true,
+            idempotent: true,
+            timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
+        });
         const rs = await s.executeQuery(yql, params, undefined, settings);
         const rowset = rs.resultSets?.[0];
         const rows = (rowset?.rows ?? []);

package/dist/routes/points.js

@@ -2,6 +2,8 @@ import { Router } from "express";
 import { upsertPoints, searchPoints, queryPoints, deletePoints, } from "../services/PointsService.js";
 import { QdrantServiceError } from "../services/errors.js";
 import { logger } from "../logging/logger.js";
+import { isCompilationTimeoutError } from "../ydb/client.js";
+import { scheduleExit } from "../utils/exit.js";
 export const pointsRouter = Router();
 // Qdrant-compatible: PUT /collections/:collection/points (upsert)
 pointsRouter.put("/:collection/points", async (req, res) => {
@@ -18,8 +20,14 @@ pointsRouter.put("/:collection/points", async (req, res) => {
         if (err instanceof QdrantServiceError) {
             return res.status(err.statusCode).json(err.payload);
         }
-        logger.error({ err }, "upsert points (PUT) failed");
         const errorMessage = err instanceof Error ? err.message : String(err);
+        if (isCompilationTimeoutError(err)) {
+            logger.error({ err }, "YDB compilation error during upsert points (PUT); scheduling process exit");
+            res.status(500).json({ status: "error", error: errorMessage });
+            scheduleExit(1);
+            return;
+        }
+        logger.error({ err }, "upsert points (PUT) failed");
         res.status(500).json({ status: "error", error: errorMessage });
     }
 });
@@ -37,8 +45,14 @@ pointsRouter.post("/:collection/points/upsert", async (req, res) => {
         if (err instanceof QdrantServiceError) {
             return res.status(err.statusCode).json(err.payload);
         }
-        logger.error({ err }, "upsert points failed");
         const errorMessage = err instanceof Error ? err.message : String(err);
+        if (isCompilationTimeoutError(err)) {
+            logger.error({ err }, "YDB compilation error during upsert points; scheduling process exit");
+            res.status(500).json({ status: "error", error: errorMessage });
+            scheduleExit(1);
+            return;
+        }
+        logger.error({ err }, "upsert points failed");
         res.status(500).json({ status: "error", error: errorMessage });
     }
 });
@@ -56,8 +70,14 @@ pointsRouter.post("/:collection/points/search", async (req, res) => {
         if (err instanceof QdrantServiceError) {
             return res.status(err.statusCode).json(err.payload);
         }
-        logger.error({ err }, "search points failed");
         const errorMessage = err instanceof Error ? err.message : String(err);
+        if (isCompilationTimeoutError(err)) {
+            logger.error({ err }, "YDB compilation error during search points; scheduling process exit");
+            res.status(500).json({ status: "error", error: errorMessage });
+            scheduleExit(1);
+            return;
+        }
+        logger.error({ err }, "search points failed");
         res.status(500).json({ status: "error", error: errorMessage });
     }
 });
@@ -76,8 +96,14 @@ pointsRouter.post("/:collection/points/query", async (req, res) => {
         if (err instanceof QdrantServiceError) {
             return res.status(err.statusCode).json(err.payload);
         }
-        logger.error({ err }, "search points (query) failed");
         const errorMessage = err instanceof Error ? err.message : String(err);
+        if (isCompilationTimeoutError(err)) {
+            logger.error({ err }, "YDB compilation error during search points (query); scheduling process exit");
+            res.status(500).json({ status: "error", error: errorMessage });
+            scheduleExit(1);
+            return;
+        }
+        logger.error({ err }, "search points (query) failed");
         res.status(500).json({ status: "error", error: errorMessage });
     }
 });

package/dist/server.js

@@ -2,11 +2,28 @@ import express from "express";
 import { collectionsRouter } from "./routes/collections.js";
 import { pointsRouter } from "./routes/points.js";
 import { requestLogger } from "./middleware/requestLogger.js";
-import { isYdbAvailable } from "./ydb/client.js";
+import { isYdbAvailable, isCompilationTimeoutError } from "./ydb/client.js";
+import { verifyCollectionsQueryCompilationForStartup } from "./repositories/collectionsRepo.js";
+import { logger } from "./logging/logger.js";
+import { scheduleExit } from "./utils/exit.js";
 export async function healthHandler(_req, res) {
     const ok = await isYdbAvailable();
     if (!ok) {
+        logger.error("YDB unavailable during health check; scheduling process exit");
         res.status(503).json({ status: "error", error: "YDB unavailable" });
+        scheduleExit(1);
+        return;
+    }
+    try {
+        await verifyCollectionsQueryCompilationForStartup();
+    }
+    catch (err) {
+        const isTimeout = isCompilationTimeoutError(err);
+        logger.error({ err }, isTimeout
+            ? "YDB compilation timeout during health probe; scheduling process exit"
+            : "YDB health probe failed; scheduling process exit");
+        res.status(503).json({ status: "error", error: "YDB health probe failed" });
+        scheduleExit(1);
         return;
     }
     res.json({ status: "ok" });

package/dist/utils/exit.d.ts

@@ -0,0 +1,2 @@
+export declare function scheduleExit(code: number): void;
+export declare function __setExitFnForTests(fn: (code: number) => void): void;

package/dist/utils/exit.js

@@ -0,0 +1,12 @@
+let exitFn = (code) => {
+    // Use process.exit in production; this will be overridden in tests.
+    process.exit(code);
+};
+export function scheduleExit(code) {
+    // Schedule exit on the next tick so HTTP responses can be flushed first.
+    setImmediate(() => exitFn(code));
+}
+// Test-only: allow overriding the underlying exit behavior.
+export function __setExitFnForTests(fn) {
+    exitFn = fn;
+}

package/dist/utils/retry.js

@@ -32,7 +32,14 @@ export async function withRetry(fn, options = {}) {
                 throw e;
             }
             const backoffMs = Math.floor(baseDelayMs * Math.pow(2, attempt) + Math.random() * 100);
-            logger.warn({ ...context, attempt, backoffMs }, "operation aborted due to transient error; retrying");
+            logger.warn({
+                ...context,
+                attempt,
+                backoffMs,
+                err: e instanceof Error
+                    ? e
+                    : new Error(typeof e === "string" ? e : JSON.stringify(e)),
+            }, "operation aborted due to transient error; retrying");
             await new Promise((r) => setTimeout(r, backoffMs));
             attempt += 1;
         }

package/dist/ydb/client.d.ts

@@ -1,6 +1,6 @@
 import type { Session, IAuthService, ExecuteQuerySettings as YdbExecuteQuerySettings } from "ydb-sdk";
-declare const Types: typeof import("ydb-sdk").Types, TypedValues: typeof import("ydb-sdk").TypedValues, TableDescription: typeof import("ydb-sdk").TableDescription, Column: typeof import("ydb-sdk").Column, ExecuteQuerySettings: typeof YdbExecuteQuerySettings;
-export { Types, TypedValues, TableDescription, Column, ExecuteQuerySettings };
+declare const Types: typeof import("ydb-sdk").Types, TypedValues: typeof import("ydb-sdk").TypedValues, TableDescription: typeof import("ydb-sdk").TableDescription, Column: typeof import("ydb-sdk").Column, ExecuteQuerySettings: typeof YdbExecuteQuerySettings, Ydb: typeof import("ydb-sdk-proto").Ydb;
+export { Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, Ydb, };
 export declare function createExecuteQuerySettings(options?: {
     keepInCache?: boolean;
     idempotent?: boolean;

package/dist/ydb/client.js

@@ -1,9 +1,9 @@
 import { createRequire } from "module";
-import { YDB_DATABASE, YDB_ENDPOINT, SESSION_POOL_MIN_SIZE, SESSION_POOL_MAX_SIZE, SESSION_KEEPALIVE_PERIOD_MS, } from "../config/env.js";
+import { YDB_DATABASE, YDB_ENDPOINT, SESSION_POOL_MIN_SIZE, SESSION_POOL_MAX_SIZE, SESSION_KEEPALIVE_PERIOD_MS, STARTUP_PROBE_SESSION_TIMEOUT_MS, } from "../config/env.js";
 import { logger } from "../logging/logger.js";
 const require = createRequire(import.meta.url);
-const { Driver, getCredentialsFromEnv, Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, OperationParams, } = require("ydb-sdk");
-export { Types, TypedValues, TableDescription, Column, ExecuteQuerySettings };
+const { Driver, getCredentialsFromEnv, Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, OperationParams, Ydb, } = require("ydb-sdk");
+export { Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, Ydb, };
 export function createExecuteQuerySettings(options) {
     const { keepInCache = true, idempotent = true } = options ?? {};
     const settings = new ExecuteQuerySettings();
@@ -30,7 +30,6 @@ const DRIVER_READY_TIMEOUT_MS = 15000;
 const TABLE_SESSION_TIMEOUT_MS = 20000;
 const YDB_HEALTHCHECK_READY_TIMEOUT_MS = 5000;
 const DRIVER_REFRESH_COOLDOWN_MS = 30000;
-const STARTUP_PROBE_SESSION_TIMEOUT_MS = 3000;
 let overrideConfig;
 let driver;
 let lastDriverRefreshAt = 0;

package/dist/ydb/schema.js

@@ -1,4 +1,4 @@
-import { withSession, TableDescription, Column, Types } from "./client.js";
+import { withSession, TableDescription, Column, Types, Ydb } from "./client.js";
 import { logger } from "../logging/logger.js";
 import { GLOBAL_POINTS_AUTOMIGRATE_ENABLED } from "../config/env.js";
 export const GLOBAL_POINTS_TABLE = "qdrant_all_points";
@@ -41,10 +41,16 @@ export async function ensureGlobalPointsTable() {
             tableDescription = await s.describeTable(GLOBAL_POINTS_TABLE);
         }
         catch {
-            // Table doesn't exist, create it with all columns using the new schema.
+            // Table doesn't exist, create it with all columns using the new schema and
+            // auto-partitioning enabled.
             const desc = new TableDescription()
                 .withColumns(new Column("uid", Types.UTF8), new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("embedding_quantized", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
                 .withPrimaryKeys("uid", "point_id");
+            desc.withPartitioningSettings({
+                partitioningByLoad: Ydb.FeatureFlag.Status.ENABLED,
+                partitioningBySize: Ydb.FeatureFlag.Status.ENABLED,
+                partitionSizeMb: 100,
+            });
             await s.createTable(GLOBAL_POINTS_TABLE, desc);
             globalPointsTableReady = true;
             logger.info(`created global points table ${GLOBAL_POINTS_TABLE}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ydb-qdrant",
-  "version": "5.0.0",
+  "version": "5.2.0",
   "main": "dist/package/api.js",
   "types": "dist/package/api.d.ts",
   "exports": {
@@ -50,6 +50,14 @@
   "author": "",
   "license": "Apache-2.0",
   "description": "Qdrant-compatible Node.js/TypeScript API that stores/searches embeddings in YDB using a global one-table layout with exact and approximate KNN search over serialized vectors.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/astandrik/ydb-qdrant.git"
+  },
+  "homepage": "https://github.com/astandrik/ydb-qdrant#readme",
+  "bugs": {
+    "url": "https://github.com/astandrik/ydb-qdrant/issues"
+  },
   "type": "module",
   "publishConfig": {
     "access": "public"
@@ -83,4 +91,4 @@
     "typescript-eslint": "^8.47.0",
     "vitest": "^4.0.12"
   }
-}
+}