ydb-qdrant 6.0.0 → 8.1.0

package/dist/ydb/schema.js

@@ -1,65 +1,91 @@
 import { withSession, TableDescription, Column, Types, Ydb } from "./client.js";
 import { logger } from "../logging/logger.js";
-import { GLOBAL_POINTS_AUTOMIGRATE_ENABLED } from "../config/env.js";
 export const GLOBAL_POINTS_TABLE = "qdrant_all_points";
 // Shared YDB-related constants for repositories.
 export { UPSERT_BATCH_SIZE } from "../config/env.js";
+let metaTableReady = false;
+let metaTableReadyInFlight = null;
 let globalPointsTableReady = false;
+let globalPointsTableReadyInFlight = null;
 function throwMigrationRequired(message) {
     logger.error(message);
     throw new Error(message);
 }
-export async function ensureMetaTable() {
-    try {
-        await withSession(async (s) => {
-            // If table exists, describeTable will succeed
-            try {
-                const tableDescription = await s.describeTable("qdr__collections");
-                const columns = tableDescription.columns ?? [];
-                const hasLastAccessedAt = columns.some((col) => col.name === "last_accessed_at");
-                if (!hasLastAccessedAt) {
-                    const alterDdl = `
-            ALTER TABLE qdr__collections
-            ADD COLUMN last_accessed_at Timestamp;
-          `;
-                    // NOTE: ydb-sdk's public TableSession type does not surface executeSchemeQuery,
-                    // but the underlying implementation provides it. This cast relies on the
-                    // current ydb-sdk internals (tested with ydb-sdk v5.11.1) to run ALTER TABLE
-                    // as a scheme query. If the SDK changes its internal API, this may need to be
-                    // revisited or replaced with an officially supported migration mechanism.
-                    const rawSession = s;
-                    await rawSession.api.executeSchemeQuery({
-                        sessionId: rawSession.sessionId,
-                        yqlText: alterDdl,
-                    });
-                    logger.info("added last_accessed_at column to metadata table qdr__collections");
-                }
-                return;
-            }
-            catch {
-                // create via schema API
-                const desc = new TableDescription()
-                    .withColumns(new Column("collection", Types.UTF8), new Column("table_name", Types.UTF8), new Column("vector_dimension", Types.UINT32), new Column("distance", Types.UTF8), new Column("vector_type", Types.UTF8), new Column("created_at", Types.TIMESTAMP), new Column("last_accessed_at", Types.TIMESTAMP))
-                    .withPrimaryKey("collection");
-                await s.createTable("qdr__collections", desc);
-                logger.info("created metadata table qdr__collections");
-            }
-        });
+function isTableNotFoundError(err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    const ctorName = err instanceof Error
+        ? err.constructor?.name
+        : undefined;
+    const statusCodeMatch = /code\s+(\d{6})/i.exec(msg);
+    const statusCode = statusCodeMatch && statusCodeMatch[1]
+        ? Number(statusCodeMatch[1])
+        : undefined;
+    // ydb-sdk exposes dedicated error classes with server status codes.
+    // In practice, table-not-found can surface as:
+    // - NotFound (code 400140)
+    // - SchemeError (code 400070) with empty issues (observed in CI logs for describeTable)
+    if (ctorName === "NotFound" || statusCode === 400140) {
+        return true;
     }
-    catch (err) {
-        logger.warn({ err }, "ensureMetaTable: failed to verify or migrate qdr__collections; subsequent operations may fail if schema is incomplete");
+    if ((ctorName === "SchemeError" || statusCode === 400070) &&
+        /:\s*\[\s*\]\s*$/i.test(msg)) {
+        return true;
     }
+    return (/table.*not found/i.test(msg) ||
+        /path.*not found/i.test(msg) ||
+        /does not exist/i.test(msg));
 }
-export async function ensureGlobalPointsTable() {
-    if (globalPointsTableReady) {
+function isAlreadyExistsError(err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return /already exists/i.test(msg) || /path.*exists/i.test(msg);
+}
+async function ensureMetaTableOnce() {
+    await withSession(async (s) => {
+        let tableDescription = null;
+        try {
+            tableDescription = await s.describeTable("qdr__collections");
+        }
+        catch (err) {
+            if (isTableNotFoundError(err)) {
+                throwMigrationRequired("Metadata table qdr__collections does not exist; please create it before starting the service");
+            }
+            throw err;
+        }
+        // Table exists: validate required columns.
+        const columns = tableDescription.columns ?? [];
+        const hasLastAccessedAt = columns.some((col) => col.name === "last_accessed_at");
+        if (!hasLastAccessedAt) {
+            throwMigrationRequired("Metadata table qdr__collections is missing required column last_accessed_at; please recreate the table or apply a manual schema migration before starting the service");
+        }
+    });
+    metaTableReady = true;
+}
+export async function ensureMetaTable() {
+    if (metaTableReady) {
         return;
     }
+    if (metaTableReadyInFlight) {
+        await metaTableReadyInFlight;
+        return;
+    }
+    metaTableReadyInFlight = ensureMetaTableOnce();
+    try {
+        await metaTableReadyInFlight;
+    }
+    finally {
+        metaTableReadyInFlight = null;
+    }
+}
+async function ensureGlobalPointsTableOnce() {
     await withSession(async (s) => {
         let tableDescription = null;
         try {
             tableDescription = await s.describeTable(GLOBAL_POINTS_TABLE);
         }
-        catch {
+        catch (err) {
+            if (!isTableNotFoundError(err)) {
+                throw err;
+            }
             // Table doesn't exist, create it with all columns using the new schema and
             // auto-partitioning enabled.
             const desc = new TableDescription()
@@ -70,35 +96,43 @@ export async function ensureGlobalPointsTable() {
                 partitioningBySize: Ydb.FeatureFlag.Status.ENABLED,
                 partitionSizeMb: 100,
             });
-            await s.createTable(GLOBAL_POINTS_TABLE, desc);
-            globalPointsTableReady = true;
-            logger.info(`created global points table ${GLOBAL_POINTS_TABLE}`);
-            return;
+            try {
+                await s.createTable(GLOBAL_POINTS_TABLE, desc);
+                logger.info(`created global points table ${GLOBAL_POINTS_TABLE}`);
+                return;
+            }
+            catch (createErr) {
+                // Race-safe: another concurrent caller might have created the table.
+                if (!isAlreadyExistsError(createErr)) {
+                    throw createErr;
+                }
+            }
+            // If the table already exists (race), fall through to a fresh describe +
+            // schema validation.
+            tableDescription = await s.describeTable(GLOBAL_POINTS_TABLE);
         }
         // Table exists, require the new embedding_quantized column.
         const columns = tableDescription.columns ?? [];
         const hasEmbeddingQuantized = columns.some((col) => col.name === "embedding_quantized");
         if (!hasEmbeddingQuantized) {
-            if (!GLOBAL_POINTS_AUTOMIGRATE_ENABLED) {
-                throwMigrationRequired(`Global points table ${GLOBAL_POINTS_TABLE} is missing required column embedding_quantized; apply the migration (e.g., ALTER TABLE ${GLOBAL_POINTS_TABLE} RENAME COLUMN embedding_bit TO embedding_quantized) or set YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE=true after backup to allow automatic migration.`);
-            }
-            const alterDdl = `
-          ALTER TABLE ${GLOBAL_POINTS_TABLE}
-          ADD COLUMN embedding_quantized String;
-        `;
-            // NOTE: Same rationale as in ensureMetaTable: executeSchemeQuery is not part of
-            // the public TableSession TypeScript surface, so we reach into the underlying
-            // ydb-sdk implementation (verified with ydb-sdk v5.11.1) to apply schema changes.
-            // If future SDK versions alter this shape, this cast and migration path must be
-            // updated accordingly.
-            const rawSession = s;
-            await rawSession.api.executeSchemeQuery({
-                sessionId: rawSession.sessionId,
-                yqlText: alterDdl,
-            });
-            logger.info(`added embedding_quantized column to existing table ${GLOBAL_POINTS_TABLE}`);
+            throwMigrationRequired(`Global points table ${GLOBAL_POINTS_TABLE} is missing required column embedding_quantized; please recreate the table or apply a manual schema migration before starting the service`);
         }
-        // Mark table ready after schema checks/migrations succeed.
-        globalPointsTableReady = true;
     });
 }
+export async function ensureGlobalPointsTable() {
+    if (globalPointsTableReady) {
+        return;
+    }
+    if (globalPointsTableReadyInFlight) {
+        await globalPointsTableReadyInFlight;
+        return;
+    }
+    globalPointsTableReadyInFlight = ensureGlobalPointsTableOnce();
+    try {
+        await globalPointsTableReadyInFlight;
+        globalPointsTableReady = true;
+    }
+    finally {
+        globalPointsTableReadyInFlight = null;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ydb-qdrant",
-  "version": "6.0.0",
+  "version": "8.1.0",
   "main": "dist/package/api.js",
   "types": "dist/package/api.d.ts",
   "exports": {
@@ -65,11 +65,8 @@
   "dependencies": {
     "@bufbuild/protobuf": "^2.10.0",
     "@grpc/grpc-js": "^1.14.0",
+    "@qdrant/js-client-rest": "^1.16.2",
     "@yandex-cloud/nodejs-sdk": "^2.9.0",
-    "@ydbjs/api": "^6.0.4",
-    "@ydbjs/core": "^6.0.4",
-    "@ydbjs/query": "^6.0.4",
-    "@ydbjs/value": "^6.0.4",
     "dotenv": "^17.2.3",
     "express": "^5.1.0",
     "nice-grpc": "^2.1.13",
@@ -91,4 +88,4 @@
     "typescript-eslint": "^8.47.0",
     "vitest": "^4.0.12"
   }
-}
+}