ydb-qdrant 4.1.0 → 4.1.1

package/README.md

@@ -402,11 +402,11 @@ curl -X POST http://localhost:8080/collections/mycol/points/delete \
 ## Notes
 - Storage layout:
   - **multi_table** (default): one YDB table per collection; metadata is tracked in `qdr__collections`.
-  - **one_table**: a single global table `qdrant_all_points` with `(uid, point_id)` PK, where `uid` encodes tenant+collection.
+  - **one_table**: a single global table `qdrant_all_points` with `(uid, point_id)` PK, where `uid` encodes tenant+collection. Columns: `uid Utf8`, `point_id Utf8`, `embedding String` (binary float), `embedding_bit String` (bit‑quantized), `payload JsonDocument`.
 - Per‑collection table schema (multi_table): `point_id Utf8` (PK), `embedding String` (binary), `payload JsonDocument`.
 - Vectors are serialized with `Knn::ToBinaryStringFloat`.
 - Search uses a single-phase top‑k over `embedding` with automatic YDB vector index (`emb_idx`) when available; falls back to table scan if missing.
-- **Vector index auto-build** (multi_table mode only): After ≥100 points upserted + 5s quiet window, a `vector_kmeans_tree` index (levels=1, clusters=128) is built automatically. Incremental updates (<100 points) skip index rebuild. In one_table mode, vector indexes are not supported and all searches use table scans.
+- **Vector index auto-build** (multi_table mode only): After ≥100 points upserted + 5s quiet window, a `vector_kmeans_tree` index (levels=1, clusters=128) is built automatically. Incremental updates (<100 points) skip index rebuild. In one_table mode, vector indexes are not supported; searches use a two‑phase approximate+exact flow over `qdrant_all_points` (bit‑quantized candidates via `embedding_bit` using the corresponding distance function, then exact re‑ranking over `embedding`). Note: For Dot metric, Phase 1 uses CosineDistance as a proxy since there is no direct distance equivalent for inner product on bit vectors.
 - **Concurrency**: During index rebuilds, YDB may return transient `Aborted`/schema metadata errors. Upserts include bounded retries with backoff to handle this automatically.
 - Filters are not yet modeled; can be added if needed.
 

package/dist/repositories/pointsRepo.one-table.js

@@ -1,7 +1,7 @@
-import { TypedValues, withSession } from "../ydb/client.js";
+import { TypedValues, Types, withSession } from "../ydb/client.js";
 import { buildJsonOrEmpty, buildVectorParam } from "../ydb/helpers.js";
 import { notifyUpsert } from "../indexing/IndexScheduler.js";
-import { mapDistanceToKnnFn } from "../utils/distance.js";
+import { mapDistanceToKnnFn, mapDistanceToBitKnnFn, } from "../utils/distance.js";
 import { withRetry, isTransientYdbError } from "../utils/retry.js";
 export async function upsertPointsOneTable(tableName, points, dimension, uid) {
     let upserted = 0;
@@ -16,11 +16,12 @@ export async function upsertPointsOneTable(tableName, points, dimension, uid) {
         DECLARE $id AS Utf8;
         DECLARE $vec AS List<Float>;
         DECLARE $payload AS JsonDocument;
-        UPSERT INTO ${tableName} (uid, point_id, embedding, payload)
+        UPSERT INTO ${tableName} (uid, point_id, embedding, embedding_bit, payload)
         VALUES (
           $uid,
           $id,
           Untag(Knn::ToBinaryStringFloat($vec), "FloatVector"),
+          Untag(Knn::ToBinaryStringBit($vec), "BitVector"),
           $payload
         );
       `;
@@ -45,50 +46,83 @@ export async function searchPointsOneTable(tableName, queryVector, top, withPayl
         throw new Error(`Vector dimension mismatch: got ${queryVector.length}, expected ${dimension}`);
     }
     const { fn, order } = mapDistanceToKnnFn(distance);
+    const { fn: bitFn, order: bitOrder } = mapDistanceToBitKnnFn(distance);
     const qf = buildVectorParam(queryVector);
-    const params = {
-        $qf: qf,
-        $k2: TypedValues.uint32(top),
-        $uid: TypedValues.utf8(uid),
-    };
-    const query = `
-    DECLARE $qf AS List<Float>;
-    DECLARE $k2 AS Uint32;
-    DECLARE $uid AS Utf8;
-    $qbinf = Knn::ToBinaryStringFloat($qf);
-    SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
-    FROM ${tableName}
-    WHERE uid = $uid
-    ORDER BY score ${order}
-    LIMIT $k2;
-  `;
-    const rs = await withSession(async (s) => {
-        return await s.executeQuery(query, params);
-    });
-    const rowset = rs.resultSets?.[0];
-    const rows = (rowset?.rows ?? []);
-    return rows.map((row) => {
-        const id = row.items?.[0]?.textValue;
-        if (typeof id !== "string") {
-            throw new Error("point_id is missing in YDB search result");
+    const candidateLimit = top * 10;
+    const results = await withSession(async (s) => {
+        // Phase 1: approximate candidate selection using embedding_bit
+        const phase1Query = `
+      DECLARE $qf AS List<Float>;
+      DECLARE $k AS Uint32;
+      DECLARE $uid AS Utf8;
+      $qbin_bit = Knn::ToBinaryStringBit($qf);
+      SELECT point_id
+      FROM ${tableName}
+      WHERE uid = $uid AND embedding_bit IS NOT NULL
+      ORDER BY ${bitFn}(embedding_bit, $qbin_bit) ${bitOrder}
+      LIMIT $k;
+    `;
+        const phase1Params = {
+            $qf: qf,
+            $k: TypedValues.uint32(candidateLimit),
+            $uid: TypedValues.utf8(uid),
+        };
+        const rs1 = await s.executeQuery(phase1Query, phase1Params);
+        const rowset1 = rs1.resultSets?.[0];
+        const rows1 = (rowset1?.rows ?? []);
+        const candidateIds = rows1
+            .map((row) => row.items?.[0]?.textValue)
+            .filter((id) => typeof id === "string");
+        if (candidateIds.length === 0) {
+            return [];
         }
-        let payload;
-        let scoreIdx = 1;
-        if (withPayload) {
-            const payloadText = row.items?.[1]?.textValue;
-            if (payloadText) {
-                try {
-                    payload = JSON.parse(payloadText);
-                }
-                catch {
-                    payload = undefined;
+        // Phase 2: exact re-ranking on full-precision embedding for candidates only
+        const phase2Query = `
+      DECLARE $qf AS List<Float>;
+      DECLARE $k AS Uint32;
+      DECLARE $uid AS Utf8;
+      DECLARE $ids AS List<Utf8>;
+      $qbinf = Knn::ToBinaryStringFloat($qf);
+      SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
+      FROM ${tableName}
+      WHERE uid = $uid AND point_id IN $ids
+      ORDER BY score ${order}
+      LIMIT $k;
+    `;
+        const idsParam = TypedValues.list(Types.UTF8, candidateIds);
+        const phase2Params = {
+            $qf: qf,
+            $k: TypedValues.uint32(top),
+            $uid: TypedValues.utf8(uid),
+            $ids: idsParam,
+        };
+        const rs2 = await s.executeQuery(phase2Query, phase2Params);
+        const rowset2 = rs2.resultSets?.[0];
+        const rows2 = (rowset2?.rows ?? []);
+        return rows2.map((row) => {
+            const id = row.items?.[0]?.textValue;
+            if (typeof id !== "string") {
+                throw new Error("point_id is missing in YDB search result");
+            }
+            let payload;
+            let scoreIdx = 1;
+            if (withPayload) {
+                const payloadText = row.items?.[1]?.textValue;
+                if (payloadText) {
+                    try {
+                        payload = JSON.parse(payloadText);
+                    }
+                    catch {
+                        payload = undefined;
+                    }
                 }
+                scoreIdx = 2;
             }
-            scoreIdx = 2;
-        }
-        const score = Number(row.items?.[scoreIdx]?.floatValue ?? row.items?.[scoreIdx]?.textValue);
-        return { id, score, ...(payload ? { payload } : {}) };
+            const score = Number(row.items?.[scoreIdx]?.floatValue ?? row.items?.[scoreIdx]?.textValue);
+            return { id, score, ...(payload ? { payload } : {}) };
+        });
     });
+    return results;
 }
 export async function deletePointsOneTable(tableName, ids, uid) {
     let deleted = 0;

package/dist/utils/distance.d.ts

@@ -4,3 +4,14 @@ export declare function mapDistanceToKnnFn(distance: DistanceKind): {
     order: "ASC" | "DESC";
 };
 export declare function mapDistanceToIndexParam(distance: DistanceKind): string;
+/**
+ * Maps a user-specified distance metric to a YDB Knn distance function
+ * suitable for bit-quantized vectors (Phase 1 approximate candidate selection).
+ * Always returns a distance function (lower is better, ASC ordering).
+ * For Dot, falls back to CosineDistance as a proxy since there is no
+ * direct distance equivalent for inner product.
+ */
+export declare function mapDistanceToBitKnnFn(distance: DistanceKind): {
+    fn: string;
+    order: "ASC";
+};

package/dist/utils/distance.js

@@ -26,3 +26,25 @@ export function mapDistanceToIndexParam(distance) {
             return "cosine";
     }
 }
+/**
+ * Maps a user-specified distance metric to a YDB Knn distance function
+ * suitable for bit-quantized vectors (Phase 1 approximate candidate selection).
+ * Always returns a distance function (lower is better, ASC ordering).
+ * For Dot, falls back to CosineDistance as a proxy since there is no
+ * direct distance equivalent for inner product.
+ */
+export function mapDistanceToBitKnnFn(distance) {
+    switch (distance) {
+        case "Cosine":
+            return { fn: "Knn::CosineDistance", order: "ASC" };
+        case "Dot":
+            // No direct distance equivalent; use Cosine as proxy
+            return { fn: "Knn::CosineDistance", order: "ASC" };
+        case "Euclid":
+            return { fn: "Knn::EuclideanDistance", order: "ASC" };
+        case "Manhattan":
+            return { fn: "Knn::ManhattanDistance", order: "ASC" };
+        default:
+            return { fn: "Knn::CosineDistance", order: "ASC" };
+    }
+}

package/dist/ydb/schema.js

@@ -30,19 +30,59 @@ export async function ensureGlobalPointsTable() {
     }
     try {
         await withSession(async (s) => {
+            let tableDescription = null;
             try {
-                await s.describeTable(GLOBAL_POINTS_TABLE);
-                globalPointsTableReady = true;
-                return;
+                tableDescription = await s.describeTable(GLOBAL_POINTS_TABLE);
             }
             catch {
+                // Table doesn't exist, create it with all columns
                 const desc = new TableDescription()
-                    .withColumns(new Column("uid", Types.UTF8), new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
+                    .withColumns(new Column("uid", Types.UTF8), new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("embedding_bit", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
                     .withPrimaryKeys("uid", "point_id");
                 await s.createTable(GLOBAL_POINTS_TABLE, desc);
                 globalPointsTableReady = true;
                 logger.info(`created global points table ${GLOBAL_POINTS_TABLE}`);
+                return;
+            }
+            // Table exists, check if embedding_bit column is present
+            const columns = tableDescription.columns ?? [];
+            const hasEmbeddingBit = columns.some((col) => col.name === "embedding_bit");
+            let needsBackfill = false;
+            if (!hasEmbeddingBit) {
+                // Add the missing embedding_bit column
+                const alterDdl = `
+          ALTER TABLE ${GLOBAL_POINTS_TABLE}
+          ADD COLUMN embedding_bit String;
+        `;
+                await s.executeQuery(alterDdl);
+                logger.info(`added embedding_bit column to existing table ${GLOBAL_POINTS_TABLE}`);
+                needsBackfill = true;
+            }
+            else {
+                // Column exists; check if any legacy rows still have NULL embedding_bit
+                const checkNullsDdl = `
+          SELECT 1 AS has_null
+          FROM ${GLOBAL_POINTS_TABLE}
+          WHERE embedding_bit IS NULL
+          LIMIT 1;
+        `;
+                const checkRes = await s.executeQuery(checkNullsDdl);
+                const hasNullRows = checkRes.resultSets?.[0]?.rows &&
+                    checkRes.resultSets[0].rows.length > 0;
+                needsBackfill = Boolean(hasNullRows);
+            }
+            if (needsBackfill) {
+                // Backfill existing rows: convert embedding to bit representation
+                const backfillDdl = `
+          UPDATE ${GLOBAL_POINTS_TABLE}
+          SET embedding_bit = Untag(Knn::ToBinaryStringBit(Knn::FloatFromBinaryString(embedding)), "BitVector")
+          WHERE embedding_bit IS NULL;
+        `;
+                await s.executeQuery(backfillDdl);
+                logger.info(`backfilled embedding_bit column from embedding in ${GLOBAL_POINTS_TABLE}`);
             }
+            // Mark table ready only after schema (and any required backfill) succeed
+            globalPointsTableReady = true;
         });
     }
     catch (err) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ydb-qdrant",
-  "version": "4.1.0",
+  "version": "4.1.1",
   "main": "dist/package/api.js",
   "types": "dist/package/api.d.ts",
   "exports": {