ydb-qdrant 4.1.1 → 4.1.2

package/README.md

@@ -401,8 +401,9 @@ curl -X POST http://localhost:8080/collections/mycol/points/delete \
 
 ## Notes
 - Storage layout:
-  - **multi_table** (default): one YDB table per collection; metadata is tracked in `qdr__collections`.
-  - **one_table**: a single global table `qdrant_all_points` with `(uid, point_id)` PK, where `uid` encodes tenant+collection. Columns: `uid Utf8`, `point_id Utf8`, `embedding String` (binary float), `embedding_bit String` (bit‑quantized), `payload JsonDocument`.
+- **multi_table** (default): one YDB table per collection; metadata is tracked in `qdr__collections`.
+- **one_table**: a single global table `qdrant_all_points` with `(uid, point_id)` PK, where `uid` encodes tenant+collection. Columns: `uid Utf8`, `point_id Utf8`, `embedding String` (binary float), `embedding_bit String` (bit‑quantized), `payload JsonDocument`.
+- **Schema migrations** (one_table mode): automatic schema/backfill steps for `qdrant_all_points` are disabled by default. To opt in, set `YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE=true` after backing up data; otherwise the service will error if the `embedding_bit` column is missing or needs backfill.
 - Per‑collection table schema (multi_table): `point_id Utf8` (PK), `embedding String` (binary), `payload JsonDocument`.
 - Vectors are serialized with `Knn::ToBinaryStringFloat`.
 - Search uses a single-phase top‑k over `embedding` with automatic YDB vector index (`emb_idx`) when available; falls back to table scan if missing.

package/dist/config/env.d.ts

@@ -3,6 +3,7 @@ export declare const YDB_ENDPOINT: string;
 export declare const YDB_DATABASE: string;
 export declare const PORT: number;
 export declare const LOG_LEVEL: string;
+export declare const GLOBAL_POINTS_AUTOMIGRATE_ENABLED: boolean;
 export declare const VECTOR_INDEX_BUILD_ENABLED: boolean;
 export declare enum CollectionStorageMode {
     MultiTable = "multi_table",

package/dist/config/env.js

@@ -3,6 +3,7 @@ export const YDB_ENDPOINT = process.env.YDB_ENDPOINT ?? "";
 export const YDB_DATABASE = process.env.YDB_DATABASE ?? "";
 export const PORT = process.env.PORT ? Number(process.env.PORT) : 8080;
 export const LOG_LEVEL = process.env.LOG_LEVEL ?? "info";
+export const GLOBAL_POINTS_AUTOMIGRATE_ENABLED = parseBooleanEnv(process.env.YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE, false);
 function parseBooleanEnv(value, defaultValue) {
     if (value === undefined) {
         return defaultValue;

package/dist/services/PointsService.js

@@ -5,7 +5,7 @@ import { deletePoints as repoDeletePoints, searchPoints as repoSearchPoints, ups
 import { requestIndexBuild } from "../indexing/IndexScheduler.js";
 import { logger } from "../logging/logger.js";
 import { VECTOR_INDEX_BUILD_ENABLED } from "../config/env.js";
-import { QdrantServiceError } from "./errors.js";
+import { QdrantServiceError, isVectorDimensionMismatchError, } from "./errors.js";
 import { normalizeCollectionContext, resolvePointsTableAndUid, } from "./CollectionService.js";
 import { normalizeSearchBodyForSearch, normalizeSearchBodyForQuery, } from "../utils/normalization.js";
 let loggedIndexBuildDisabled = false;
@@ -27,7 +27,25 @@ export async function upsertPoints(ctx, body) {
         });
     }
     const { tableName, uid } = await resolvePointsTableAndUid(normalized, meta);
-    const upserted = await repoUpsertPoints(tableName, parsed.data.points, meta.dimension, uid);
+    let upserted;
+    try {
+        upserted = await repoUpsertPoints(tableName, parsed.data.points, meta.dimension, uid);
+    }
+    catch (err) {
+        if (isVectorDimensionMismatchError(err)) {
+            logger.warn({
+                tenant: normalized.tenant,
+                collection: normalized.collection,
+                table: tableName,
+                dimension: meta.dimension,
+            }, "upsertPoints: vector dimension mismatch");
+            throw new QdrantServiceError(400, {
+                status: "error",
+                error: err.message,
+            });
+        }
+        throw err;
+    }
     if (VECTOR_INDEX_BUILD_ENABLED) {
         requestIndexBuild(tableName, meta.dimension, meta.distance, meta.vectorType);
     }
@@ -79,7 +97,26 @@ async function executeSearch(ctx, normalizedSearch, source) {
         distance: meta.distance,
         vectorType: meta.vectorType,
     }, `${source}: executing`);
-    const hits = await repoSearchPoints(tableName, parsed.data.vector, parsed.data.top, parsed.data.with_payload, meta.distance, meta.dimension, uid);
+    let hits;
+    try {
+        hits = await repoSearchPoints(tableName, parsed.data.vector, parsed.data.top, parsed.data.with_payload, meta.distance, meta.dimension, uid);
+    }
+    catch (err) {
+        if (isVectorDimensionMismatchError(err)) {
+            logger.warn({
+                tenant: normalized.tenant,
+                collection: normalized.collection,
+                table: tableName,
+                dimension: meta.dimension,
+                queryVectorLen: parsed.data.vector.length,
+            }, `${source}: vector dimension mismatch`);
+            throw new QdrantServiceError(400, {
+                status: "error",
+                error: err.message,
+            });
+        }
+        throw err;
+    }
     const threshold = normalizedSearch.scoreThreshold;
     const filtered = threshold === undefined
         ? hits

package/dist/services/errors.d.ts

@@ -2,6 +2,7 @@ export interface QdrantServiceErrorPayload {
     status: "error";
     error: unknown;
 }
+export declare function isVectorDimensionMismatchError(err: unknown): err is Error;
 export declare class QdrantServiceError extends Error {
     readonly statusCode: number;
     readonly payload: QdrantServiceErrorPayload;

package/dist/services/errors.js

@@ -1,3 +1,6 @@
+export function isVectorDimensionMismatchError(err) {
+    return (err instanceof Error && err.message.startsWith("Vector dimension mismatch"));
+}
 export class QdrantServiceError extends Error {
     statusCode;
     payload;

package/dist/ydb/schema.js

@@ -1,7 +1,12 @@
 import { withSession, TableDescription, Column, Types } from "./client.js";
 import { logger } from "../logging/logger.js";
+import { GLOBAL_POINTS_AUTOMIGRATE_ENABLED } from "../config/env.js";
 export const GLOBAL_POINTS_TABLE = "qdrant_all_points";
 let globalPointsTableReady = false;
+function throwMigrationRequired(message) {
+    logger.error(message);
+    throw new Error(message);
+}
 export async function ensureMetaTable() {
     try {
         await withSession(async (s) => {
@@ -28,64 +33,61 @@ export async function ensureGlobalPointsTable() {
     if (globalPointsTableReady) {
         return;
     }
-    try {
-        await withSession(async (s) => {
-            let tableDescription = null;
-            try {
-                tableDescription = await s.describeTable(GLOBAL_POINTS_TABLE);
-            }
-            catch {
-                // Table doesn't exist, create it with all columns
-                const desc = new TableDescription()
-                    .withColumns(new Column("uid", Types.UTF8), new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("embedding_bit", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
-                    .withPrimaryKeys("uid", "point_id");
-                await s.createTable(GLOBAL_POINTS_TABLE, desc);
-                globalPointsTableReady = true;
-                logger.info(`created global points table ${GLOBAL_POINTS_TABLE}`);
-                return;
+    await withSession(async (s) => {
+        let tableDescription = null;
+        try {
+            tableDescription = await s.describeTable(GLOBAL_POINTS_TABLE);
+        }
+        catch {
+            // Table doesn't exist, create it with all columns
+            const desc = new TableDescription()
+                .withColumns(new Column("uid", Types.UTF8), new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("embedding_bit", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
+                .withPrimaryKeys("uid", "point_id");
+            await s.createTable(GLOBAL_POINTS_TABLE, desc);
+            globalPointsTableReady = true;
+            logger.info(`created global points table ${GLOBAL_POINTS_TABLE}`);
+            return;
+        }
+        // Table exists, check if embedding_bit column is present
+        const columns = tableDescription.columns ?? [];
+        const hasEmbeddingBit = columns.some((col) => col.name === "embedding_bit");
+        let needsBackfill = false;
+        if (!hasEmbeddingBit) {
+            if (!GLOBAL_POINTS_AUTOMIGRATE_ENABLED) {
+                throwMigrationRequired(`Global points table ${GLOBAL_POINTS_TABLE} is missing required column embedding_bit; set YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE=true after backup to apply the migration manually.`);
             }
-            // Table exists, check if embedding_bit column is present
-            const columns = tableDescription.columns ?? [];
-            const hasEmbeddingBit = columns.some((col) => col.name === "embedding_bit");
-            let needsBackfill = false;
-            if (!hasEmbeddingBit) {
-                // Add the missing embedding_bit column
-                const alterDdl = `
+            const alterDdl = `
           ALTER TABLE ${GLOBAL_POINTS_TABLE}
           ADD COLUMN embedding_bit String;
         `;
-                await s.executeQuery(alterDdl);
-                logger.info(`added embedding_bit column to existing table ${GLOBAL_POINTS_TABLE}`);
-                needsBackfill = true;
-            }
-            else {
-                // Column exists; check if any legacy rows still have NULL embedding_bit
-                const checkNullsDdl = `
+            await s.executeQuery(alterDdl);
+            logger.info(`added embedding_bit column to existing table ${GLOBAL_POINTS_TABLE}`);
+            needsBackfill = true;
+        }
+        else {
+            const checkNullsDdl = `
           SELECT 1 AS has_null
           FROM ${GLOBAL_POINTS_TABLE}
           WHERE embedding_bit IS NULL
           LIMIT 1;
         `;
-                const checkRes = await s.executeQuery(checkNullsDdl);
-                const hasNullRows = checkRes.resultSets?.[0]?.rows &&
-                    checkRes.resultSets[0].rows.length > 0;
-                needsBackfill = Boolean(hasNullRows);
+            const checkRes = await s.executeQuery(checkNullsDdl);
+            const rows = checkRes?.resultSets?.[0]?.rows ?? [];
+            needsBackfill = rows.length > 0;
+        }
+        if (needsBackfill) {
+            if (!GLOBAL_POINTS_AUTOMIGRATE_ENABLED) {
+                throwMigrationRequired(`Global points table ${GLOBAL_POINTS_TABLE} requires backfill for embedding_bit; set YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE=true after backup to apply the migration manually.`);
             }
-            if (needsBackfill) {
-                // Backfill existing rows: convert embedding to bit representation
-                const backfillDdl = `
+            const backfillDdl = `
           UPDATE ${GLOBAL_POINTS_TABLE}
           SET embedding_bit = Untag(Knn::ToBinaryStringBit(Knn::FloatFromBinaryString(embedding)), "BitVector")
           WHERE embedding_bit IS NULL;
         `;
-                await s.executeQuery(backfillDdl);
-                logger.info(`backfilled embedding_bit column from embedding in ${GLOBAL_POINTS_TABLE}`);
-            }
-            // Mark table ready only after schema (and any required backfill) succeed
-            globalPointsTableReady = true;
-        });
-    }
-    catch (err) {
-        logger.debug({ err }, "ensureGlobalPointsTable: ignored");
-    }
+            await s.executeQuery(backfillDdl);
+            logger.info(`backfilled embedding_bit column from embedding in ${GLOBAL_POINTS_TABLE}`);
+        }
+        // Mark table ready only after schema (and any required backfill) succeed
+        globalPointsTableReady = true;
+    });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ydb-qdrant",
-  "version": "4.1.1",
+  "version": "4.1.2",
   "main": "dist/package/api.js",
   "types": "dist/package/api.d.ts",
   "exports": {