ydb-qdrant 4.1.0 → 4.1.2

package/README.md

@@ -401,12 +401,13 @@ curl -X POST http://localhost:8080/collections/mycol/points/delete \
 
 ## Notes
 - Storage layout:
-  - **multi_table** (default): one YDB table per collection; metadata is tracked in `qdr__collections`.
-  - **one_table**: a single global table `qdrant_all_points` with `(uid, point_id)` PK, where `uid` encodes tenant+collection.
+- **multi_table** (default): one YDB table per collection; metadata is tracked in `qdr__collections`.
+- **one_table**: a single global table `qdrant_all_points` with `(uid, point_id)` PK, where `uid` encodes tenant+collection. Columns: `uid Utf8`, `point_id Utf8`, `embedding String` (binary float), `embedding_bit String` (bit‑quantized), `payload JsonDocument`.
+- **Schema migrations** (one_table mode): automatic schema/backfill steps for `qdrant_all_points` are disabled by default. To opt in, set `YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE=true` after backing up data; otherwise the service will error if the `embedding_bit` column is missing or needs backfill.
 - Per‑collection table schema (multi_table): `point_id Utf8` (PK), `embedding String` (binary), `payload JsonDocument`.
 - Vectors are serialized with `Knn::ToBinaryStringFloat`.
 - Search uses a single-phase top‑k over `embedding` with automatic YDB vector index (`emb_idx`) when available; falls back to table scan if missing.
-- **Vector index auto-build** (multi_table mode only): After ≥100 points upserted + 5s quiet window, a `vector_kmeans_tree` index (levels=1, clusters=128) is built automatically. Incremental updates (<100 points) skip index rebuild. In one_table mode, vector indexes are not supported and all searches use table scans.
+- **Vector index auto-build** (multi_table mode only): After ≥100 points upserted + 5s quiet window, a `vector_kmeans_tree` index (levels=1, clusters=128) is built automatically. Incremental updates (<100 points) skip index rebuild. In one_table mode, vector indexes are not supported; searches use a two‑phase approximate+exact flow over `qdrant_all_points` (bit‑quantized candidates via `embedding_bit` using the corresponding distance function, then exact re‑ranking over `embedding`). Note: For Dot metric, Phase 1 uses CosineDistance as a proxy since there is no direct distance equivalent for inner product on bit vectors.
 - **Concurrency**: During index rebuilds, YDB may return transient `Aborted`/schema metadata errors. Upserts include bounded retries with backoff to handle this automatically.
 - Filters are not yet modeled; can be added if needed.
 

package/dist/config/env.d.ts

@@ -3,6 +3,7 @@ export declare const YDB_ENDPOINT: string;
 export declare const YDB_DATABASE: string;
 export declare const PORT: number;
 export declare const LOG_LEVEL: string;
+export declare const GLOBAL_POINTS_AUTOMIGRATE_ENABLED: boolean;
 export declare const VECTOR_INDEX_BUILD_ENABLED: boolean;
 export declare enum CollectionStorageMode {
     MultiTable = "multi_table",

package/dist/config/env.js

@@ -3,6 +3,7 @@ export const YDB_ENDPOINT = process.env.YDB_ENDPOINT ?? "";
 export const YDB_DATABASE = process.env.YDB_DATABASE ?? "";
 export const PORT = process.env.PORT ? Number(process.env.PORT) : 8080;
 export const LOG_LEVEL = process.env.LOG_LEVEL ?? "info";
+export const GLOBAL_POINTS_AUTOMIGRATE_ENABLED = parseBooleanEnv(process.env.YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE, false);
 function parseBooleanEnv(value, defaultValue) {
     if (value === undefined) {
         return defaultValue;

package/dist/repositories/pointsRepo.one-table.js

@@ -1,7 +1,7 @@
-import { TypedValues, withSession } from "../ydb/client.js";
+import { TypedValues, Types, withSession } from "../ydb/client.js";
 import { buildJsonOrEmpty, buildVectorParam } from "../ydb/helpers.js";
 import { notifyUpsert } from "../indexing/IndexScheduler.js";
-import { mapDistanceToKnnFn } from "../utils/distance.js";
+import { mapDistanceToKnnFn, mapDistanceToBitKnnFn, } from "../utils/distance.js";
 import { withRetry, isTransientYdbError } from "../utils/retry.js";
 export async function upsertPointsOneTable(tableName, points, dimension, uid) {
     let upserted = 0;
@@ -16,11 +16,12 @@ export async function upsertPointsOneTable(tableName, points, dimension, uid) {
         DECLARE $id AS Utf8;
         DECLARE $vec AS List<Float>;
         DECLARE $payload AS JsonDocument;
-        UPSERT INTO ${tableName} (uid, point_id, embedding, payload)
+        UPSERT INTO ${tableName} (uid, point_id, embedding, embedding_bit, payload)
         VALUES (
           $uid,
           $id,
           Untag(Knn::ToBinaryStringFloat($vec), "FloatVector"),
+          Untag(Knn::ToBinaryStringBit($vec), "BitVector"),
           $payload
         );
       `;
@@ -45,50 +46,83 @@ export async function searchPointsOneTable(tableName, queryVector, top, withPayl
         throw new Error(`Vector dimension mismatch: got ${queryVector.length}, expected ${dimension}`);
     }
     const { fn, order } = mapDistanceToKnnFn(distance);
+    const { fn: bitFn, order: bitOrder } = mapDistanceToBitKnnFn(distance);
     const qf = buildVectorParam(queryVector);
-    const params = {
-        $qf: qf,
-        $k2: TypedValues.uint32(top),
-        $uid: TypedValues.utf8(uid),
-    };
-    const query = `
-    DECLARE $qf AS List<Float>;
-    DECLARE $k2 AS Uint32;
-    DECLARE $uid AS Utf8;
-    $qbinf = Knn::ToBinaryStringFloat($qf);
-    SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
-    FROM ${tableName}
-    WHERE uid = $uid
-    ORDER BY score ${order}
-    LIMIT $k2;
-  `;
-    const rs = await withSession(async (s) => {
-        return await s.executeQuery(query, params);
-    });
-    const rowset = rs.resultSets?.[0];
-    const rows = (rowset?.rows ?? []);
-    return rows.map((row) => {
-        const id = row.items?.[0]?.textValue;
-        if (typeof id !== "string") {
-            throw new Error("point_id is missing in YDB search result");
+    const candidateLimit = top * 10;
+    const results = await withSession(async (s) => {
+        // Phase 1: approximate candidate selection using embedding_bit
+        const phase1Query = `
+      DECLARE $qf AS List<Float>;
+      DECLARE $k AS Uint32;
+      DECLARE $uid AS Utf8;
+      $qbin_bit = Knn::ToBinaryStringBit($qf);
+      SELECT point_id
+      FROM ${tableName}
+      WHERE uid = $uid AND embedding_bit IS NOT NULL
+      ORDER BY ${bitFn}(embedding_bit, $qbin_bit) ${bitOrder}
+      LIMIT $k;
+    `;
+        const phase1Params = {
+            $qf: qf,
+            $k: TypedValues.uint32(candidateLimit),
+            $uid: TypedValues.utf8(uid),
+        };
+        const rs1 = await s.executeQuery(phase1Query, phase1Params);
+        const rowset1 = rs1.resultSets?.[0];
+        const rows1 = (rowset1?.rows ?? []);
+        const candidateIds = rows1
+            .map((row) => row.items?.[0]?.textValue)
+            .filter((id) => typeof id === "string");
+        if (candidateIds.length === 0) {
+            return [];
         }
-        let payload;
-        let scoreIdx = 1;
-        if (withPayload) {
-            const payloadText = row.items?.[1]?.textValue;
-            if (payloadText) {
-                try {
-                    payload = JSON.parse(payloadText);
-                }
-                catch {
-                    payload = undefined;
+        // Phase 2: exact re-ranking on full-precision embedding for candidates only
+        const phase2Query = `
+      DECLARE $qf AS List<Float>;
+      DECLARE $k AS Uint32;
+      DECLARE $uid AS Utf8;
+      DECLARE $ids AS List<Utf8>;
+      $qbinf = Knn::ToBinaryStringFloat($qf);
+      SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
+      FROM ${tableName}
+      WHERE uid = $uid AND point_id IN $ids
+      ORDER BY score ${order}
+      LIMIT $k;
+    `;
+        const idsParam = TypedValues.list(Types.UTF8, candidateIds);
+        const phase2Params = {
+            $qf: qf,
+            $k: TypedValues.uint32(top),
+            $uid: TypedValues.utf8(uid),
+            $ids: idsParam,
+        };
+        const rs2 = await s.executeQuery(phase2Query, phase2Params);
+        const rowset2 = rs2.resultSets?.[0];
+        const rows2 = (rowset2?.rows ?? []);
+        return rows2.map((row) => {
+            const id = row.items?.[0]?.textValue;
+            if (typeof id !== "string") {
+                throw new Error("point_id is missing in YDB search result");
+            }
+            let payload;
+            let scoreIdx = 1;
+            if (withPayload) {
+                const payloadText = row.items?.[1]?.textValue;
+                if (payloadText) {
+                    try {
+                        payload = JSON.parse(payloadText);
+                    }
+                    catch {
+                        payload = undefined;
+                    }
                 }
+                scoreIdx = 2;
             }
-            scoreIdx = 2;
-        }
-        const score = Number(row.items?.[scoreIdx]?.floatValue ?? row.items?.[scoreIdx]?.textValue);
-        return { id, score, ...(payload ? { payload } : {}) };
+            const score = Number(row.items?.[scoreIdx]?.floatValue ?? row.items?.[scoreIdx]?.textValue);
+            return { id, score, ...(payload ? { payload } : {}) };
+        });
     });
+    return results;
 }
 export async function deletePointsOneTable(tableName, ids, uid) {
     let deleted = 0;

package/dist/services/PointsService.js

@@ -5,7 +5,7 @@ import { deletePoints as repoDeletePoints, searchPoints as repoSearchPoints, ups
 import { requestIndexBuild } from "../indexing/IndexScheduler.js";
 import { logger } from "../logging/logger.js";
 import { VECTOR_INDEX_BUILD_ENABLED } from "../config/env.js";
-import { QdrantServiceError } from "./errors.js";
+import { QdrantServiceError, isVectorDimensionMismatchError, } from "./errors.js";
 import { normalizeCollectionContext, resolvePointsTableAndUid, } from "./CollectionService.js";
 import { normalizeSearchBodyForSearch, normalizeSearchBodyForQuery, } from "../utils/normalization.js";
 let loggedIndexBuildDisabled = false;
@@ -27,7 +27,25 @@ export async function upsertPoints(ctx, body) {
         });
     }
     const { tableName, uid } = await resolvePointsTableAndUid(normalized, meta);
-    const upserted = await repoUpsertPoints(tableName, parsed.data.points, meta.dimension, uid);
+    let upserted;
+    try {
+        upserted = await repoUpsertPoints(tableName, parsed.data.points, meta.dimension, uid);
+    }
+    catch (err) {
+        if (isVectorDimensionMismatchError(err)) {
+            logger.warn({
+                tenant: normalized.tenant,
+                collection: normalized.collection,
+                table: tableName,
+                dimension: meta.dimension,
+            }, "upsertPoints: vector dimension mismatch");
+            throw new QdrantServiceError(400, {
+                status: "error",
+                error: err.message,
+            });
+        }
+        throw err;
+    }
     if (VECTOR_INDEX_BUILD_ENABLED) {
         requestIndexBuild(tableName, meta.dimension, meta.distance, meta.vectorType);
     }
@@ -79,7 +97,26 @@ async function executeSearch(ctx, normalizedSearch, source) {
         distance: meta.distance,
         vectorType: meta.vectorType,
     }, `${source}: executing`);
-    const hits = await repoSearchPoints(tableName, parsed.data.vector, parsed.data.top, parsed.data.with_payload, meta.distance, meta.dimension, uid);
+    let hits;
+    try {
+        hits = await repoSearchPoints(tableName, parsed.data.vector, parsed.data.top, parsed.data.with_payload, meta.distance, meta.dimension, uid);
+    }
+    catch (err) {
+        if (isVectorDimensionMismatchError(err)) {
+            logger.warn({
+                tenant: normalized.tenant,
+                collection: normalized.collection,
+                table: tableName,
+                dimension: meta.dimension,
+                queryVectorLen: parsed.data.vector.length,
+            }, `${source}: vector dimension mismatch`);
+            throw new QdrantServiceError(400, {
+                status: "error",
+                error: err.message,
+            });
+        }
+        throw err;
+    }
     const threshold = normalizedSearch.scoreThreshold;
     const filtered = threshold === undefined
         ? hits

package/dist/services/errors.d.ts

@@ -2,6 +2,7 @@ export interface QdrantServiceErrorPayload {
     status: "error";
     error: unknown;
 }
+export declare function isVectorDimensionMismatchError(err: unknown): err is Error;
 export declare class QdrantServiceError extends Error {
     readonly statusCode: number;
     readonly payload: QdrantServiceErrorPayload;

package/dist/services/errors.js

@@ -1,3 +1,6 @@
+export function isVectorDimensionMismatchError(err) {
+    return (err instanceof Error && err.message.startsWith("Vector dimension mismatch"));
+}
 export class QdrantServiceError extends Error {
     statusCode;
     payload;

package/dist/utils/distance.d.ts

@@ -4,3 +4,14 @@ export declare function mapDistanceToKnnFn(distance: DistanceKind): {
     order: "ASC" | "DESC";
 };
 export declare function mapDistanceToIndexParam(distance: DistanceKind): string;
+/**
+ * Maps a user-specified distance metric to a YDB Knn distance function
+ * suitable for bit-quantized vectors (Phase 1 approximate candidate selection).
+ * Always returns a distance function (lower is better, ASC ordering).
+ * For Dot, falls back to CosineDistance as a proxy since there is no
+ * direct distance equivalent for inner product.
+ */
+export declare function mapDistanceToBitKnnFn(distance: DistanceKind): {
+    fn: string;
+    order: "ASC";
+};

package/dist/utils/distance.js

@@ -26,3 +26,25 @@ export function mapDistanceToIndexParam(distance) {
             return "cosine";
     }
 }
+/**
+ * Maps a user-specified distance metric to a YDB Knn distance function
+ * suitable for bit-quantized vectors (Phase 1 approximate candidate selection).
+ * Always returns a distance function (lower is better, ASC ordering).
+ * For Dot, falls back to CosineDistance as a proxy since there is no
+ * direct distance equivalent for inner product.
+ */
+export function mapDistanceToBitKnnFn(distance) {
+    switch (distance) {
+        case "Cosine":
+            return { fn: "Knn::CosineDistance", order: "ASC" };
+        case "Dot":
+            // No direct distance equivalent; use Cosine as proxy
+            return { fn: "Knn::CosineDistance", order: "ASC" };
+        case "Euclid":
+            return { fn: "Knn::EuclideanDistance", order: "ASC" };
+        case "Manhattan":
+            return { fn: "Knn::ManhattanDistance", order: "ASC" };
+        default:
+            return { fn: "Knn::CosineDistance", order: "ASC" };
+    }
+}

package/dist/ydb/schema.js

@@ -1,7 +1,12 @@
 import { withSession, TableDescription, Column, Types } from "./client.js";
 import { logger } from "../logging/logger.js";
+import { GLOBAL_POINTS_AUTOMIGRATE_ENABLED } from "../config/env.js";
 export const GLOBAL_POINTS_TABLE = "qdrant_all_points";
 let globalPointsTableReady = false;
+function throwMigrationRequired(message) {
+    logger.error(message);
+    throw new Error(message);
+}
 export async function ensureMetaTable() {
     try {
         await withSession(async (s) => {
@@ -28,24 +33,61 @@ export async function ensureGlobalPointsTable() {
     if (globalPointsTableReady) {
         return;
     }
-    try {
-        await withSession(async (s) => {
-            try {
-                await s.describeTable(GLOBAL_POINTS_TABLE);
-                globalPointsTableReady = true;
-                return;
+    await withSession(async (s) => {
+        let tableDescription = null;
+        try {
+            tableDescription = await s.describeTable(GLOBAL_POINTS_TABLE);
+        }
+        catch {
+            // Table doesn't exist, create it with all columns
+            const desc = new TableDescription()
+                .withColumns(new Column("uid", Types.UTF8), new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("embedding_bit", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
+                .withPrimaryKeys("uid", "point_id");
+            await s.createTable(GLOBAL_POINTS_TABLE, desc);
+            globalPointsTableReady = true;
+            logger.info(`created global points table ${GLOBAL_POINTS_TABLE}`);
+            return;
+        }
+        // Table exists, check if embedding_bit column is present
+        const columns = tableDescription.columns ?? [];
+        const hasEmbeddingBit = columns.some((col) => col.name === "embedding_bit");
+        let needsBackfill = false;
+        if (!hasEmbeddingBit) {
+            if (!GLOBAL_POINTS_AUTOMIGRATE_ENABLED) {
+                throwMigrationRequired(`Global points table ${GLOBAL_POINTS_TABLE} is missing required column embedding_bit; set YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE=true after backup to apply the migration manually.`);
             }
-            catch {
-                const desc = new TableDescription()
-                    .withColumns(new Column("uid", Types.UTF8), new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
-                    .withPrimaryKeys("uid", "point_id");
-                await s.createTable(GLOBAL_POINTS_TABLE, desc);
-                globalPointsTableReady = true;
-                logger.info(`created global points table ${GLOBAL_POINTS_TABLE}`);
+            const alterDdl = `
+          ALTER TABLE ${GLOBAL_POINTS_TABLE}
+          ADD COLUMN embedding_bit String;
+        `;
+            await s.executeQuery(alterDdl);
+            logger.info(`added embedding_bit column to existing table ${GLOBAL_POINTS_TABLE}`);
+            needsBackfill = true;
+        }
+        else {
+            const checkNullsDdl = `
+          SELECT 1 AS has_null
+          FROM ${GLOBAL_POINTS_TABLE}
+          WHERE embedding_bit IS NULL
+          LIMIT 1;
+        `;
+            const checkRes = await s.executeQuery(checkNullsDdl);
+            const rows = checkRes?.resultSets?.[0]?.rows ?? [];
+            needsBackfill = rows.length > 0;
+        }
+        if (needsBackfill) {
+            if (!GLOBAL_POINTS_AUTOMIGRATE_ENABLED) {
+                throwMigrationRequired(`Global points table ${GLOBAL_POINTS_TABLE} requires backfill for embedding_bit; set YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE=true after backup to apply the migration manually.`);
             }
-        });
-    }
-    catch (err) {
-        logger.debug({ err }, "ensureGlobalPointsTable: ignored");
-    }
+            const backfillDdl = `
+          UPDATE ${GLOBAL_POINTS_TABLE}
+          SET embedding_bit = Untag(Knn::ToBinaryStringBit(Knn::FloatFromBinaryString(embedding)), "BitVector")
+          WHERE embedding_bit IS NULL;
+        `;
+            await s.executeQuery(backfillDdl);
+            logger.info(`backfilled embedding_bit column from embedding in ${GLOBAL_POINTS_TABLE}`);
+        }
+        // Mark table ready only after schema (and any required backfill) succeed
+        globalPointsTableReady = true;
+    });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ydb-qdrant",
-  "version": "4.1.0",
+  "version": "4.1.2",
   "main": "dist/package/api.js",
   "types": "dist/package/api.d.ts",
   "exports": {