ydb-qdrant 2.2.2 → 3.0.0

package/README.md

@@ -400,7 +400,7 @@ curl -X POST http://localhost:8080/collections/mycol/points/delete \
 ## Notes
 - One YDB table is created per collection; metadata is tracked in table `qdr__collections`.
 - Each collection table schema: `point_id Utf8` (PK), `embedding String` (binary), `payload JsonDocument`.
-- Vectors are serialized with `Knn::ToBinaryStringFloat` (or `Knn::ToBinaryStringUint8` if collection uses uint8).
+- Vectors are serialized with `Knn::ToBinaryStringFloat`.
 - Search uses a single-phase top‑k over `embedding` with automatic YDB vector index (`emb_idx`) when available; falls back to table scan if missing.
 - **Vector index auto-build**: After ≥100 points upserted + 5s quiet window, a `vector_kmeans_tree` index (levels=1, clusters=128) is built automatically. Incremental updates (<100 points) skip index rebuild.
 - **Concurrency**: During index rebuilds, YDB may return transient `Aborted`/schema metadata errors. Upserts include bounded retries with backoff to handle this automatically.

package/dist/config/env.d.ts

@@ -3,4 +3,4 @@ export declare const YDB_ENDPOINT: string;
 export declare const YDB_DATABASE: string;
 export declare const PORT: number;
 export declare const LOG_LEVEL: string;
-export declare const APPROX_PRESELECT: number;
+export declare const VECTOR_INDEX_BUILD_ENABLED: boolean;

package/dist/config/env.js

@@ -3,6 +3,18 @@ export const YDB_ENDPOINT = process.env.YDB_ENDPOINT ?? "";
 export const YDB_DATABASE = process.env.YDB_DATABASE ?? "";
 export const PORT = process.env.PORT ? Number(process.env.PORT) : 8080;
 export const LOG_LEVEL = process.env.LOG_LEVEL ?? "info";
-export const APPROX_PRESELECT = process.env.APPROX_PRESELECT
-    ? Math.max(1, Math.min(5000, Number(process.env.APPROX_PRESELECT)))
-    : 1000;
+function parseBooleanEnv(value, defaultValue) {
+    if (value === undefined) {
+        return defaultValue;
+    }
+    const normalized = value.trim().toLowerCase();
+    if (normalized === "" ||
+        normalized === "0" ||
+        normalized === "false" ||
+        normalized === "no" ||
+        normalized === "off") {
+        return false;
+    }
+    return true;
+}
+export const VECTOR_INDEX_BUILD_ENABLED = parseBooleanEnv(process.env.VECTOR_INDEX_BUILD_ENABLED, false);

package/dist/repositories/pointsRepo.d.ts

@@ -1,10 +1,10 @@
-import type { VectorType, DistanceKind } from "../types";
+import type { DistanceKind } from "../types";
 export declare function upsertPoints(tableName: string, points: Array<{
     id: string | number;
     vector: number[];
     payload?: Record<string, unknown>;
-}>, vectorType: VectorType, dimension: number): Promise<number>;
-export declare function searchPoints(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, vectorType: VectorType, dimension: number): Promise<Array<{
+}>, dimension: number): Promise<number>;
+export declare function searchPoints(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number): Promise<Array<{
     id: string;
     score: number;
     payload?: Record<string, unknown>;

package/dist/repositories/pointsRepo.js

@@ -2,7 +2,8 @@ import { TypedValues, withSession } from "../ydb/client.js";
 import { buildJsonOrEmpty, buildVectorParam } from "../ydb/helpers.js";
 import { logger } from "../logging/logger.js";
 import { notifyUpsert } from "../indexing/IndexScheduler.js";
-export async function upsertPoints(tableName, points, vectorType, dimension) {
+import { VECTOR_INDEX_BUILD_ENABLED } from "../config/env.js";
+export async function upsertPoints(tableName, points, dimension) {
     let upserted = 0;
     await withSession(async (s) => {
         for (const p of points) {
@@ -12,18 +13,18 @@ export async function upsertPoints(tableName, points, vectorType, dimension) {
             }
             const ddl = `
         DECLARE $id AS Utf8;
-        DECLARE $vec AS List<${vectorType === "uint8" ? "Uint8" : "Float"}>;
+        DECLARE $vec AS List<Float>;
         DECLARE $payload AS JsonDocument;
         UPSERT INTO ${tableName} (point_id, embedding, payload)
         VALUES (
           $id,
-          Untag(Knn::ToBinaryString${vectorType === "uint8" ? "Uint8" : "Float"}($vec), "${vectorType === "uint8" ? "Uint8Vector" : "FloatVector"}"),
+          Untag(Knn::ToBinaryStringFloat($vec), "FloatVector"),
           $payload
         );
       `;
             const params = {
                 $id: TypedValues.utf8(id),
-                $vec: buildVectorParam(p.vector, vectorType),
+                $vec: buildVectorParam(p.vector),
                 $payload: buildJsonOrEmpty(p.payload),
             };
             // Retry on transient schema/metadata mismatches during index rebuild
@@ -55,47 +56,55 @@ export async function upsertPoints(tableName, points, vectorType, dimension) {
     return upserted;
 }
 // Removed legacy index backfill helper
-export async function searchPoints(tableName, queryVector, top, withPayload, distance, vectorType, dimension) {
+export async function searchPoints(tableName, queryVector, top, withPayload, distance, dimension) {
     if (queryVector.length !== dimension) {
         throw new Error(`Vector dimension mismatch: got ${queryVector.length}, expected ${dimension}`);
     }
     const { fn, order } = mapDistanceToKnnFn(distance);
     // Single-phase search over embedding using vector index if present
-    const qf = buildVectorParam(queryVector, vectorType);
+    const qf = buildVectorParam(queryVector);
     const params = {
         $qf: qf,
         $k2: TypedValues.uint32(top),
     };
     const buildQuery = (useIndex) => `
-    DECLARE $qf AS List<${vectorType === "uint8" ? "Uint8" : "Float"}>;
+    DECLARE $qf AS List<Float>;
     DECLARE $k2 AS Uint32;
-    $qbinf = Knn::ToBinaryString${vectorType === "uint8" ? "Uint8" : "Float"}($qf);
+    $qbinf = Knn::ToBinaryStringFloat($qf);
     SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
     FROM ${tableName}${useIndex ? " VIEW emb_idx" : ""}
     ORDER BY score ${order}
     LIMIT $k2;
   `;
     let rs;
-    try {
-        // Try with vector index first
-        rs = await withSession(async (s) => {
-            return await s.executeQuery(buildQuery(true), params);
-        });
-        logger.info({ tableName }, "vector index found; using index for search");
-    }
-    catch (e) {
-        const msg = e instanceof Error ? e.message : String(e);
-        // Fallback to table scan if index not found or not ready
-        if (/not found|does not exist|no such index|no global index|is not ready to use/i.test(msg)) {
-            logger.info({ tableName }, "vector index not available (missing or building); falling back to table scan");
+    if (VECTOR_INDEX_BUILD_ENABLED) {
+        try {
+            // Try with vector index first
             rs = await withSession(async (s) => {
-                return await s.executeQuery(buildQuery(false), params);
+                return await s.executeQuery(buildQuery(true), params);
             });
+            logger.info({ tableName }, "vector index found; using index for search");
         }
-        else {
-            throw e;
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            const indexUnavailable = /not found|does not exist|no such index|no global index|is not ready to use/i.test(msg);
+            if (indexUnavailable) {
+                logger.info({ tableName }, "vector index not available (missing or building); falling back to table scan");
+                rs = await withSession(async (s) => {
+                    return await s.executeQuery(buildQuery(false), params);
+                });
+            }
+            else {
+                throw e;
+            }
         }
     }
+    else {
+        // Vector index usage disabled: always use table scan
+        rs = await withSession(async (s) => {
+            return await s.executeQuery(buildQuery(false), params);
+        });
+    }
     const rowset = rs.resultSets?.[0];
     const rows = (rowset?.rows ?? []);
     return rows.map((row) => {

package/dist/services/QdrantService.js

@@ -5,6 +5,7 @@ import { createCollection as repoCreateCollection, deleteCollection as repoDelet
 import { deletePoints as repoDeletePoints, searchPoints as repoSearchPoints, upsertPoints as repoUpsertPoints, } from "../repositories/pointsRepo.js";
 import { requestIndexBuild } from "../indexing/IndexScheduler.js";
 import { logger } from "../logging/logger.js";
+import { VECTOR_INDEX_BUILD_ENABLED } from "../config/env.js";
 export class QdrantServiceError extends Error {
     statusCode;
     payload;
@@ -87,6 +88,7 @@ export async function deleteCollection(ctx) {
     await repoDeleteCollection(normalized.metaKey);
     return { acknowledged: true };
 }
+let loggedIndexBuildDisabled = false;
 function isNumberArray(value) {
     return Array.isArray(value) && value.every((x) => typeof x === "number");
 }
@@ -212,8 +214,14 @@ export async function upsertPoints(ctx, body) {
             error: parsed.error.flatten(),
         });
     }
-    const upserted = await repoUpsertPoints(meta.table, parsed.data.points, meta.vectorType, meta.dimension);
-    requestIndexBuild(meta.table, meta.dimension, meta.distance, meta.vectorType);
+    const upserted = await repoUpsertPoints(meta.table, parsed.data.points, meta.dimension);
+    if (VECTOR_INDEX_BUILD_ENABLED) {
+        requestIndexBuild(meta.table, meta.dimension, meta.distance, meta.vectorType);
+    }
+    else if (!loggedIndexBuildDisabled) {
+        logger.info({ table: meta.table }, "vector index building disabled by env; skipping automatic emb_idx rebuilds");
+        loggedIndexBuildDisabled = true;
+    }
     return { upserted };
 }
 async function executeSearch(ctx, normalizedSearch, source) {
@@ -257,7 +265,7 @@ async function executeSearch(ctx, normalizedSearch, source) {
         distance: meta.distance,
         vectorType: meta.vectorType,
     }, `${source}: executing`);
-    const hits = await repoSearchPoints(meta.table, parsed.data.vector, parsed.data.top, parsed.data.with_payload, meta.distance, meta.vectorType, meta.dimension);
+    const hits = await repoSearchPoints(meta.table, parsed.data.vector, parsed.data.top, parsed.data.with_payload, meta.distance, meta.dimension);
     const threshold = normalizedSearch.scoreThreshold;
     const filtered = threshold === undefined
         ? hits

package/dist/types.d.ts

@@ -1,13 +1,12 @@
 import { z } from "zod";
 export type DistanceKind = "Cosine" | "Euclid" | "Dot" | "Manhattan";
-export type VectorType = "float" | "uint8";
+export type VectorType = "float";
 export declare const CreateCollectionReq: z.ZodObject<{
     vectors: z.ZodObject<{
         size: z.ZodNumber;
         distance: z.ZodType<DistanceKind>;
         data_type: z.ZodOptional<z.ZodEnum<{
             float: "float";
-            uint8: "uint8";
         }>>;
     }, z.core.$strip>;
 }, z.core.$strip>;

package/dist/types.js

@@ -8,7 +8,7 @@ export const CreateCollectionReq = z.object({
             "Dot",
             "Manhattan",
         ]),
-        data_type: z.enum(["float", "uint8"]).optional(),
+        data_type: z.enum(["float"]).optional(),
     }),
 });
 export const UpsertPointsReq = z.object({

package/dist/ydb/helpers.d.ts

@@ -1,2 +1,2 @@
-export declare function buildVectorParam(vector: number[], vectorType: "float" | "uint8"): import("ydb-sdk-proto").Ydb.ITypedValue;
+export declare function buildVectorParam(vector: number[]): import("ydb-sdk-proto").Ydb.ITypedValue;
 export declare function buildJsonOrEmpty(payload?: Record<string, unknown>): import("ydb-sdk-proto").Ydb.ITypedValue;

package/dist/ydb/helpers.js

@@ -1,44 +1,6 @@
 import { Types, TypedValues } from "./client.js";
-export function buildVectorParam(vector, vectorType) {
-    let list;
-    if (vectorType === "uint8") {
-        // Check if vector is already quantized (integers in [0,255])
-        const isAlreadyQuantized = vector.every(v => Number.isInteger(v) && v >= 0 && v <= 255);
-        if (isAlreadyQuantized) {
-            list = vector;
-        }
-        else {
-            // Float embeddings need quantization. Per YDB docs (knn.md lines 282-294):
-            // Formula: ((x - min) / (max - min)) * 255
-            const min = Math.min(...vector);
-            const max = Math.max(...vector);
-            // Determine quantization strategy based on detected range
-            if (min >= 0 && max <= 1.01) {
-                // Normalized [0,1] embeddings (common for some models)
-                list = vector.map(v => Math.round(Math.max(0, Math.min(1, v)) * 255));
-            }
-            else if (min >= -1.01 && max <= 1.01) {
-                // Normalized [-1,1] embeddings (most common)
-                // Map to [0,255]: ((x + 1) / 2) * 255 = (x + 1) * 127.5
-                list = vector.map(v => Math.round((Math.max(-1, Math.min(1, v)) + 1) * 127.5));
-            }
-            else {
-                // General case: linear scaling from [min,max] to [0,255]
-                const range = max - min;
-                if (range > 0) {
-                    list = vector.map(v => Math.round(((v - min) / range) * 255));
-                }
-                else {
-                    // All values identical; map to midpoint
-                    list = vector.map(() => 127);
-                }
-            }
-        }
-    }
-    else {
-        list = vector;
-    }
-    return TypedValues.list(vectorType === "uint8" ? Types.UINT8 : Types.FLOAT, list);
+export function buildVectorParam(vector) {
+    return TypedValues.list(Types.FLOAT, vector);
 }
 export function buildJsonOrEmpty(payload) {
     return TypedValues.jsonDocument(JSON.stringify(payload ?? {}));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ydb-qdrant",
-  "version": "2.2.2",
+  "version": "3.0.0",
   "main": "dist/package/Api.js",
   "types": "dist/package/Api.d.ts",
   "exports": {
@@ -16,7 +16,7 @@
   "scripts": {
     "test": "vitest run --exclude \"test/integration/**\"",
     "test:coverage": "vitest run --coverage --exclude \"test/integration/**\"",
-    "test:integration": "vitest run test/integration/YdbRealIntegration.test.ts",
+    "test:integration": "VECTOR_INDEX_BUILD_ENABLED=false vitest run test/integration/YdbRealIntegration.index-disabled.test.ts && VECTOR_INDEX_BUILD_ENABLED=true vitest run test/integration/YdbRealIntegration.test.ts",
     "build": "tsc -p tsconfig.json",
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "dev": "tsx watch src/index.ts",
@@ -46,7 +46,7 @@
   ],
   "author": "",
   "license": "ISC",
-  "description": "Qdrant-compatible Node.js/TypeScript API that stores/searches embeddings in YDB using approximate coarse-to-fine vector search (quantized uint8 preselect + float refine).",
+  "description": "Qdrant-compatible Node.js/TypeScript API that stores/searches embeddings in YDB using single-phase top-k vector search with an automatic vector index and table-scan fallback.",
   "type": "module",
   "publishConfig": {
     "access": "public"