ydb-qdrant 5.0.0 → 5.1.0

package/dist/config/env.d.ts

@@ -16,3 +16,5 @@ export declare const UPSERT_BATCH_SIZE: number;
 export declare const SESSION_POOL_MIN_SIZE: number;
 export declare const SESSION_POOL_MAX_SIZE: number;
 export declare const SESSION_KEEPALIVE_PERIOD_MS: number;
+export declare const UPSERT_OPERATION_TIMEOUT_MS: number;
+export declare const SEARCH_OPERATION_TIMEOUT_MS: number;

package/dist/config/env.js

@@ -67,3 +67,5 @@ const NORMALIZED_SESSION_POOL_MIN_SIZE = RAW_SESSION_POOL_MIN_SIZE > RAW_SESSION
 export const SESSION_POOL_MIN_SIZE = NORMALIZED_SESSION_POOL_MIN_SIZE;
 export const SESSION_POOL_MAX_SIZE = RAW_SESSION_POOL_MAX_SIZE;
 export const SESSION_KEEPALIVE_PERIOD_MS = parseIntegerEnv(process.env.YDB_SESSION_KEEPALIVE_PERIOD_MS, 5000, { min: 1000, max: 60000 });
+export const UPSERT_OPERATION_TIMEOUT_MS = parseIntegerEnv(process.env.YDB_QDRANT_UPSERT_TIMEOUT_MS, 5000, { min: 1000 });
+export const SEARCH_OPERATION_TIMEOUT_MS = parseIntegerEnv(process.env.YDB_QDRANT_SEARCH_TIMEOUT_MS, 10000, { min: 1000 });

package/dist/repositories/collectionsRepo.js

@@ -1,6 +1,7 @@
 import { TypedValues, withSession, createExecuteQuerySettings, withStartupProbeSession, createExecuteQuerySettingsWithTimeout, } from "../ydb/client.js";
 import { uidFor } from "../utils/tenant.js";
 import { createCollectionOneTable, deleteCollectionOneTable, } from "./collectionsRepo.one-table.js";
+import { withRetry, isTransientYdbError } from "../utils/retry.js";
 export async function createCollection(metaKey, dim, distance, vectorType) {
     await createCollectionOneTable(metaKey, dim, distance, vectorType);
 }
@@ -35,15 +36,22 @@ export async function verifyCollectionsQueryCompilationForStartup() {
     FROM qdr__collections
     WHERE collection = $collection;
   `;
-    await withStartupProbeSession(async (s) => {
-        const settings = createExecuteQuerySettingsWithTimeout({
-            keepInCache: true,
-            idempotent: true,
-            timeoutMs: 3000,
+    await withRetry(async () => {
+        await withStartupProbeSession(async (s) => {
+            const settings = createExecuteQuerySettingsWithTimeout({
+                keepInCache: true,
+                idempotent: true,
+                timeoutMs: 3000,
+            });
+            await s.executeQuery(qry, {
+                $collection: TypedValues.utf8(probeKey),
+            }, undefined, settings);
         });
-        await s.executeQuery(qry, {
-            $collection: TypedValues.utf8(probeKey),
-        }, undefined, settings);
+    }, {
+        isTransient: isTransientYdbError,
+        maxRetries: 2,
+        baseDelayMs: 200,
+        context: { probe: "collections_startup_compilation" },
     });
 }
 export async function deleteCollection(metaKey, uid) {

package/dist/repositories/pointsRepo.one-table.js

@@ -1,9 +1,9 @@
-import { TypedValues, Types, withSession, createExecuteQuerySettings, } from "../ydb/client.js";
+import { TypedValues, Types, withSession, createExecuteQuerySettings, createExecuteQuerySettingsWithTimeout, } from "../ydb/client.js";
 import { buildVectorParam, buildVectorBinaryParams } from "../ydb/helpers.js";
 import { mapDistanceToKnnFn, mapDistanceToBitKnnFn, } from "../utils/distance.js";
 import { withRetry, isTransientYdbError } from "../utils/retry.js";
 import { UPSERT_BATCH_SIZE } from "../ydb/schema.js";
-import { CLIENT_SIDE_SERIALIZATION_ENABLED, SearchMode, } from "../config/env.js";
+import { CLIENT_SIDE_SERIALIZATION_ENABLED, SearchMode, UPSERT_OPERATION_TIMEOUT_MS, SEARCH_OPERATION_TIMEOUT_MS, } from "../config/env.js";
 import { logger } from "../logging/logger.js";
 export async function upsertPointsOneTable(tableName, points, dimension, uid) {
     for (const p of points) {
@@ -24,7 +24,11 @@ export async function upsertPointsOneTable(tableName, points, dimension, uid) {
     }
     let upserted = 0;
     await withSession(async (s) => {
-        const settings = createExecuteQuerySettings();
+        const settings = createExecuteQuerySettingsWithTimeout({
+            keepInCache: true,
+            idempotent: true,
+            timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
+        });
         for (let i = 0; i < points.length; i += UPSERT_BATCH_SIZE) {
             const batch = points.slice(i, i + UPSERT_BATCH_SIZE);
             let ddl;
@@ -192,7 +196,11 @@ async function searchPointsOneTableExact(tableName, queryVector, top, withPayloa
                 vectorPreview: queryVector.slice(0, 3),
             },
         }, "one_table search (exact): executing YQL");
-        const settings = createExecuteQuerySettings();
+        const settings = createExecuteQuerySettingsWithTimeout({
+            keepInCache: true,
+            idempotent: true,
+            timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
+        });
         const rs = await s.executeQuery(yql, params, undefined, settings);
         const rowset = rs.resultSets?.[0];
         const rows = (rowset?.rows ?? []);
@@ -334,7 +342,11 @@ async function searchPointsOneTableApproximate(tableName, queryVector, top, with
                 },
             }, "one_table search (approximate): executing YQL");
         }
-        const settings = createExecuteQuerySettings();
+        const settings = createExecuteQuerySettingsWithTimeout({
+            keepInCache: true,
+            idempotent: true,
+            timeoutMs: SEARCH_OPERATION_TIMEOUT_MS,
+        });
         const rs = await s.executeQuery(yql, params, undefined, settings);
         const rowset = rs.resultSets?.[0];
         const rows = (rowset?.rows ?? []);

package/dist/routes/points.js

@@ -2,6 +2,8 @@ import { Router } from "express";
 import { upsertPoints, searchPoints, queryPoints, deletePoints, } from "../services/PointsService.js";
 import { QdrantServiceError } from "../services/errors.js";
 import { logger } from "../logging/logger.js";
+import { isCompilationTimeoutError } from "../ydb/client.js";
+import { scheduleExit } from "../utils/exit.js";
 export const pointsRouter = Router();
 // Qdrant-compatible: PUT /collections/:collection/points (upsert)
 pointsRouter.put("/:collection/points", async (req, res) => {
@@ -18,8 +20,14 @@ pointsRouter.put("/:collection/points", async (req, res) => {
         if (err instanceof QdrantServiceError) {
             return res.status(err.statusCode).json(err.payload);
         }
-        logger.error({ err }, "upsert points (PUT) failed");
         const errorMessage = err instanceof Error ? err.message : String(err);
+        if (isCompilationTimeoutError(err)) {
+            logger.error({ err }, "YDB compilation error during upsert points (PUT); scheduling process exit");
+            res.status(500).json({ status: "error", error: errorMessage });
+            scheduleExit(1);
+            return;
+        }
+        logger.error({ err }, "upsert points (PUT) failed");
         res.status(500).json({ status: "error", error: errorMessage });
     }
 });
@@ -37,8 +45,14 @@ pointsRouter.post("/:collection/points/upsert", async (req, res) => {
         if (err instanceof QdrantServiceError) {
             return res.status(err.statusCode).json(err.payload);
         }
-        logger.error({ err }, "upsert points failed");
         const errorMessage = err instanceof Error ? err.message : String(err);
+        if (isCompilationTimeoutError(err)) {
+            logger.error({ err }, "YDB compilation error during upsert points; scheduling process exit");
+            res.status(500).json({ status: "error", error: errorMessage });
+            scheduleExit(1);
+            return;
+        }
+        logger.error({ err }, "upsert points failed");
         res.status(500).json({ status: "error", error: errorMessage });
     }
 });
@@ -56,8 +70,14 @@ pointsRouter.post("/:collection/points/search", async (req, res) => {
         if (err instanceof QdrantServiceError) {
             return res.status(err.statusCode).json(err.payload);
         }
-        logger.error({ err }, "search points failed");
         const errorMessage = err instanceof Error ? err.message : String(err);
+        if (isCompilationTimeoutError(err)) {
+            logger.error({ err }, "YDB compilation error during search points; scheduling process exit");
+            res.status(500).json({ status: "error", error: errorMessage });
+            scheduleExit(1);
+            return;
+        }
+        logger.error({ err }, "search points failed");
         res.status(500).json({ status: "error", error: errorMessage });
     }
 });
@@ -76,8 +96,14 @@ pointsRouter.post("/:collection/points/query", async (req, res) => {
         if (err instanceof QdrantServiceError) {
             return res.status(err.statusCode).json(err.payload);
         }
-        logger.error({ err }, "search points (query) failed");
         const errorMessage = err instanceof Error ? err.message : String(err);
+        if (isCompilationTimeoutError(err)) {
+            logger.error({ err }, "YDB compilation error during search points (query); scheduling process exit");
+            res.status(500).json({ status: "error", error: errorMessage });
+            scheduleExit(1);
+            return;
+        }
+        logger.error({ err }, "search points (query) failed");
         res.status(500).json({ status: "error", error: errorMessage });
     }
 });

package/dist/server.js

@@ -2,11 +2,28 @@ import express from "express";
 import { collectionsRouter } from "./routes/collections.js";
 import { pointsRouter } from "./routes/points.js";
 import { requestLogger } from "./middleware/requestLogger.js";
-import { isYdbAvailable } from "./ydb/client.js";
+import { isYdbAvailable, isCompilationTimeoutError } from "./ydb/client.js";
+import { verifyCollectionsQueryCompilationForStartup } from "./repositories/collectionsRepo.js";
+import { logger } from "./logging/logger.js";
+import { scheduleExit } from "./utils/exit.js";
 export async function healthHandler(_req, res) {
     const ok = await isYdbAvailable();
     if (!ok) {
+        logger.error("YDB unavailable during health check; scheduling process exit");
         res.status(503).json({ status: "error", error: "YDB unavailable" });
+        scheduleExit(1);
+        return;
+    }
+    try {
+        await verifyCollectionsQueryCompilationForStartup();
+    }
+    catch (err) {
+        const isTimeout = isCompilationTimeoutError(err);
+        logger.error({ err }, isTimeout
+            ? "YDB compilation timeout during health probe; scheduling process exit"
+            : "YDB health probe failed; scheduling process exit");
+        res.status(503).json({ status: "error", error: "YDB health probe failed" });
+        scheduleExit(1);
         return;
     }
     res.json({ status: "ok" });

package/dist/utils/exit.d.ts

@@ -0,0 +1,2 @@
+export declare function scheduleExit(code: number): void;
+export declare function __setExitFnForTests(fn: (code: number) => void): void;

package/dist/utils/exit.js

@@ -0,0 +1,12 @@
+let exitFn = (code) => {
+    // Use process.exit in production; this will be overridden in tests.
+    process.exit(code);
+};
+export function scheduleExit(code) {
+    // Schedule exit on the next tick so HTTP responses can be flushed first.
+    setImmediate(() => exitFn(code));
+}
+// Test-only: allow overriding the underlying exit behavior.
+export function __setExitFnForTests(fn) {
+    exitFn = fn;
+}

package/dist/ydb/client.d.ts

@@ -1,6 +1,6 @@
 import type { Session, IAuthService, ExecuteQuerySettings as YdbExecuteQuerySettings } from "ydb-sdk";
-declare const Types: typeof import("ydb-sdk").Types, TypedValues: typeof import("ydb-sdk").TypedValues, TableDescription: typeof import("ydb-sdk").TableDescription, Column: typeof import("ydb-sdk").Column, ExecuteQuerySettings: typeof YdbExecuteQuerySettings;
-export { Types, TypedValues, TableDescription, Column, ExecuteQuerySettings };
+declare const Types: typeof import("ydb-sdk").Types, TypedValues: typeof import("ydb-sdk").TypedValues, TableDescription: typeof import("ydb-sdk").TableDescription, Column: typeof import("ydb-sdk").Column, ExecuteQuerySettings: typeof YdbExecuteQuerySettings, Ydb: typeof import("ydb-sdk-proto").Ydb;
+export { Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, Ydb, };
 export declare function createExecuteQuerySettings(options?: {
     keepInCache?: boolean;
     idempotent?: boolean;

package/dist/ydb/client.js

@@ -2,8 +2,8 @@ import { createRequire } from "module";
 import { YDB_DATABASE, YDB_ENDPOINT, SESSION_POOL_MIN_SIZE, SESSION_POOL_MAX_SIZE, SESSION_KEEPALIVE_PERIOD_MS, } from "../config/env.js";
 import { logger } from "../logging/logger.js";
 const require = createRequire(import.meta.url);
-const { Driver, getCredentialsFromEnv, Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, OperationParams, } = require("ydb-sdk");
-export { Types, TypedValues, TableDescription, Column, ExecuteQuerySettings };
+const { Driver, getCredentialsFromEnv, Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, OperationParams, Ydb, } = require("ydb-sdk");
+export { Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, Ydb, };
 export function createExecuteQuerySettings(options) {
     const { keepInCache = true, idempotent = true } = options ?? {};
     const settings = new ExecuteQuerySettings();

package/dist/ydb/schema.js

@@ -1,4 +1,4 @@
-import { withSession, TableDescription, Column, Types } from "./client.js";
+import { withSession, TableDescription, Column, Types, Ydb } from "./client.js";
 import { logger } from "../logging/logger.js";
 import { GLOBAL_POINTS_AUTOMIGRATE_ENABLED } from "../config/env.js";
 export const GLOBAL_POINTS_TABLE = "qdrant_all_points";
@@ -41,10 +41,16 @@ export async function ensureGlobalPointsTable() {
             tableDescription = await s.describeTable(GLOBAL_POINTS_TABLE);
         }
         catch {
-            // Table doesn't exist, create it with all columns using the new schema.
+            // Table doesn't exist, create it with all columns using the new schema and
+            // auto-partitioning enabled.
             const desc = new TableDescription()
                 .withColumns(new Column("uid", Types.UTF8), new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("embedding_quantized", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
                 .withPrimaryKeys("uid", "point_id");
+            desc.withPartitioningSettings({
+                partitioningByLoad: Ydb.FeatureFlag.Status.ENABLED,
+                partitioningBySize: Ydb.FeatureFlag.Status.ENABLED,
+                partitionSizeMb: 100,
+            });
             await s.createTable(GLOBAL_POINTS_TABLE, desc);
             globalPointsTableReady = true;
             logger.info(`created global points table ${GLOBAL_POINTS_TABLE}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ydb-qdrant",
-  "version": "5.0.0",
+  "version": "5.1.0",
   "main": "dist/package/api.js",
   "types": "dist/package/api.d.ts",
   "exports": {