ydb-qdrant 4.6.0 → 4.7.0

package/README.md

@@ -3,6 +3,8 @@
 [![Build](https://img.shields.io/github/actions/workflow/status/astandrik/ydb-qdrant/ci-build.yml?branch=main&label=build)](https://github.com/astandrik/ydb-qdrant/actions/workflows/ci-build.yml)
 [![Tests](https://img.shields.io/github/actions/workflow/status/astandrik/ydb-qdrant/ci-tests.yml?branch=main&label=tests)](https://github.com/astandrik/ydb-qdrant/actions/workflows/ci-tests.yml)
 [![Integration Tests](https://img.shields.io/github/actions/workflow/status/astandrik/ydb-qdrant/ci-integration.yml?branch=main&label=integration%20tests)](https://github.com/astandrik/ydb-qdrant/actions/workflows/ci-integration.yml)
+[![k6 Soak Load Test](https://img.shields.io/github/actions/workflow/status/astandrik/ydb-qdrant/ci-load-soak.yml?branch=main&label=k6%20soak%20load%20test)](https://github.com/astandrik/ydb-qdrant/actions/workflows/ci-load-soak.yml)
+[![k6 Stress Load Test](https://img.shields.io/github/actions/workflow/status/astandrik/ydb-qdrant/ci-load-stress.yml?branch=main&label=k6%20stress%20load%20test)](https://github.com/astandrik/ydb-qdrant/actions/workflows/ci-load-stress.yml)
 [![Coverage](https://coveralls.io/repos/github/astandrik/ydb-qdrant/badge.svg?branch=main)](https://coveralls.io/github/astandrik/ydb-qdrant?branch=main)
 
 [![Recall (multi_table)](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astandrik/ydb-qdrant/recall-badges/recall-multi-table.json)](https://github.com/astandrik/ydb-qdrant/actions/workflows/ci-recall.yml)
@@ -90,6 +92,9 @@ export PORT=8080
 export LOG_LEVEL=info
 # Collection storage mode (optional; default is multi_table)
 export YDB_QDRANT_COLLECTION_STORAGE_MODE=multi_table   # or one_table
+# One-table search tuning (one_table mode only)
+export YDB_QDRANT_SEARCH_MODE=approximate               # or exact
+export YDB_QDRANT_OVERFETCH_MULTIPLIER=10               # candidate multiplier in approximate mode
 ```
 
 ## Use as a Node.js library (npm package)

package/dist/config/env.d.ts

@@ -11,3 +11,11 @@ export declare const COLLECTION_STORAGE_MODE: CollectionStorageMode;
 export declare const GLOBAL_POINTS_AUTOMIGRATE_ENABLED: boolean;
 export declare const VECTOR_INDEX_BUILD_ENABLED: boolean;
 export declare function isOneTableMode(mode: CollectionStorageMode): mode is CollectionStorageMode.OneTable;
+export declare enum SearchMode {
+    Exact = "exact",
+    Approximate = "approximate"
+}
+export declare const SEARCH_MODE: SearchMode;
+export declare const OVERFETCH_MULTIPLIER: number;
+export declare const CLIENT_SIDE_SERIALIZATION_ENABLED: boolean;
+export declare const UPSERT_BATCH_SIZE: number;

package/dist/config/env.js

@@ -3,6 +3,23 @@ export const YDB_ENDPOINT = process.env.YDB_ENDPOINT ?? "";
 export const YDB_DATABASE = process.env.YDB_DATABASE ?? "";
 export const PORT = process.env.PORT ? Number(process.env.PORT) : 8080;
 export const LOG_LEVEL = process.env.LOG_LEVEL ?? "info";
+function parseIntegerEnv(value, defaultValue, opts) {
+    if (value === undefined) {
+        return defaultValue;
+    }
+    const parsed = Number.parseInt(value.trim(), 10);
+    if (!Number.isFinite(parsed)) {
+        return defaultValue;
+    }
+    let result = parsed;
+    if (opts?.min !== undefined && result < opts.min) {
+        result = opts.min;
+    }
+    if (opts?.max !== undefined && result > opts.max) {
+        result = opts.max;
+    }
+    return result;
+}
 function parseBooleanEnv(value, defaultValue) {
     if (value === undefined) {
         return defaultValue;
@@ -36,3 +53,28 @@ export const VECTOR_INDEX_BUILD_ENABLED = parseBooleanEnv(process.env.VECTOR_IND
 export function isOneTableMode(mode) {
     return mode === CollectionStorageMode.OneTable;
 }
+export var SearchMode;
+(function (SearchMode) {
+    SearchMode["Exact"] = "exact";
+    SearchMode["Approximate"] = "approximate";
+})(SearchMode || (SearchMode = {}));
+function resolveSearchModeEnv(mode) {
+    const raw = process.env.YDB_QDRANT_SEARCH_MODE;
+    const normalized = raw?.trim().toLowerCase();
+    if (normalized === SearchMode.Exact) {
+        return SearchMode.Exact;
+    }
+    if (normalized === SearchMode.Approximate) {
+        return SearchMode.Approximate;
+    }
+    // Default: keep current behavior for one-table (approximate two-phase search).
+    if (isOneTableMode(mode)) {
+        return SearchMode.Approximate;
+    }
+    // For multi-table, this value is currently unused but defaults to approximate.
+    return SearchMode.Approximate;
+}
+export const SEARCH_MODE = resolveSearchModeEnv(COLLECTION_STORAGE_MODE);
+export const OVERFETCH_MULTIPLIER = parseIntegerEnv(process.env.YDB_QDRANT_OVERFETCH_MULTIPLIER, 10, { min: 1 });
+export const CLIENT_SIDE_SERIALIZATION_ENABLED = parseBooleanEnv(process.env.YDB_QDRANT_CLIENT_SIDE_SERIALIZATION_ENABLED, false);
+export const UPSERT_BATCH_SIZE = parseIntegerEnv(process.env.YDB_QDRANT_UPSERT_BATCH_SIZE, 100, { min: 1 });

package/dist/repositories/pointsRepo.js

@@ -1,4 +1,5 @@
 import { upsertPointsMultiTable, searchPointsMultiTable, deletePointsMultiTable, } from "./pointsRepo.multi-table.js";
+import { SEARCH_MODE, OVERFETCH_MULTIPLIER, } from "../config/env.js";
 import { upsertPointsOneTable, searchPointsOneTable, deletePointsOneTable, } from "./pointsRepo.one-table.js";
 export async function upsertPoints(tableName, points, dimension, uid) {
     if (uid) {
@@ -7,8 +8,9 @@ export async function upsertPoints(tableName, points, dimension, uid) {
     return await upsertPointsMultiTable(tableName, points, dimension);
 }
 export async function searchPoints(tableName, queryVector, top, withPayload, distance, dimension, uid) {
+    const mode = SEARCH_MODE;
     if (uid) {
-        return await searchPointsOneTable(tableName, queryVector, top, withPayload, distance, dimension, uid);
+        return await searchPointsOneTable(tableName, queryVector, top, withPayload, distance, dimension, uid, mode, OVERFETCH_MULTIPLIER);
     }
     return await searchPointsMultiTable(tableName, queryVector, top, withPayload, distance, dimension);
 }

package/dist/repositories/pointsRepo.multi-table.js

@@ -1,39 +1,54 @@
-import { TypedValues, withSession } from "../ydb/client.js";
-import { buildJsonOrEmpty, buildVectorParam } from "../ydb/helpers.js";
+import { Types, TypedValues, withSession } from "../ydb/client.js";
+import { buildVectorParam } from "../ydb/helpers.js";
 import { logger } from "../logging/logger.js";
 import { notifyUpsert } from "../indexing/IndexScheduler.js";
 import { VECTOR_INDEX_BUILD_ENABLED } from "../config/env.js";
 import { mapDistanceToKnnFn } from "../utils/distance.js";
 import { withRetry, isTransientYdbError } from "../utils/retry.js";
+import { UPSERT_BATCH_SIZE } from "../ydb/schema.js";
 export async function upsertPointsMultiTable(tableName, points, dimension) {
+    for (const p of points) {
+        const id = String(p.id);
+        if (p.vector.length !== dimension) {
+            throw new Error(`Vector dimension mismatch for id=${id}: got ${p.vector.length}, expected ${dimension}`);
+        }
+    }
     let upserted = 0;
     await withSession(async (s) => {
-        for (const p of points) {
-            const id = String(p.id);
-            if (p.vector.length !== dimension) {
-                throw new Error(`Vector dimension mismatch for id=${id}: got ${p.vector.length}, expected ${dimension}`);
-            }
+        for (let i = 0; i < points.length; i += UPSERT_BATCH_SIZE) {
+            const batch = points.slice(i, i + UPSERT_BATCH_SIZE);
             const ddl = `
-        DECLARE $id AS Utf8;
-        DECLARE $vec AS List<Float>;
-        DECLARE $payload AS JsonDocument;
+        DECLARE $rows AS List<Struct<
+          point_id: Utf8,
+          vec: List<Float>,
+          payload: JsonDocument
+        >>;
+
         UPSERT INTO ${tableName} (point_id, embedding, payload)
-        VALUES (
-          $id,
-          Untag(Knn::ToBinaryStringFloat($vec), "FloatVector"),
-          $payload
-        );
+        SELECT
+          point_id,
+          Untag(Knn::ToBinaryStringFloat(vec), "FloatVector") AS embedding,
+          payload
+        FROM AS_TABLE($rows);
       `;
+            const rowType = Types.struct({
+                point_id: Types.UTF8,
+                vec: Types.list(Types.FLOAT),
+                payload: Types.JSON_DOCUMENT,
+            });
+            const rowsValue = TypedValues.list(rowType, batch.map((p) => ({
+                point_id: String(p.id),
+                vec: p.vector,
+                payload: JSON.stringify(p.payload ?? {}),
+            })));
             const params = {
-                $id: TypedValues.utf8(id),
-                $vec: buildVectorParam(p.vector),
-                $payload: buildJsonOrEmpty(p.payload),
+                $rows: rowsValue,
             };
             await withRetry(() => s.executeQuery(ddl, params), {
                 isTransient: isTransientYdbError,
-                context: { tableName, id },
+                context: { tableName, batchSize: batch.length },
             });
-            upserted += 1;
+            upserted += batch.length;
         }
     });
     notifyUpsert(tableName, upserted);

package/dist/repositories/pointsRepo.one-table.d.ts

@@ -1,10 +1,11 @@
 import type { DistanceKind } from "../types";
+import { SearchMode } from "../config/env.js";
 export declare function upsertPointsOneTable(tableName: string, points: Array<{
     id: string | number;
     vector: number[];
     payload?: Record<string, unknown>;
 }>, dimension: number, uid: string): Promise<number>;
-export declare function searchPointsOneTable(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number, uid: string): Promise<Array<{
+export declare function searchPointsOneTable(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number, uid: string, mode: SearchMode | undefined, overfetchMultiplier: number): Promise<Array<{
     id: string;
     score: number;
     payload?: Record<string, unknown>;

package/dist/repositories/pointsRepo.one-table.js

@@ -1,56 +1,345 @@
 import { TypedValues, Types, withSession } from "../ydb/client.js";
-import { buildJsonOrEmpty, buildVectorParam } from "../ydb/helpers.js";
+import { buildVectorParam, buildVectorBinaryParams } from "../ydb/helpers.js";
 import { notifyUpsert } from "../indexing/IndexScheduler.js";
 import { mapDistanceToKnnFn, mapDistanceToBitKnnFn, } from "../utils/distance.js";
 import { withRetry, isTransientYdbError } from "../utils/retry.js";
+import { UPSERT_BATCH_SIZE } from "../ydb/schema.js";
+import { CLIENT_SIDE_SERIALIZATION_ENABLED, SearchMode, } from "../config/env.js";
+import { logger } from "../logging/logger.js";
 export async function upsertPointsOneTable(tableName, points, dimension, uid) {
+    for (const p of points) {
+        const id = String(p.id);
+        if (p.vector.length !== dimension) {
+            const previewLength = Math.min(16, p.vector.length);
+            const vectorPreview = previewLength > 0 ? p.vector.slice(0, previewLength) : [];
+            logger.warn({
+                tableName,
+                uid,
+                pointId: id,
+                vectorLen: p.vector.length,
+                expectedDimension: dimension,
+                vectorPreview,
+            }, "upsertPointsOneTable: vector dimension mismatch");
+            throw new Error(`Vector dimension mismatch for id=${id}: got ${p.vector.length}, expected ${dimension}`);
+        }
+    }
     let upserted = 0;
     await withSession(async (s) => {
-        for (const p of points) {
-            const id = String(p.id);
-            if (p.vector.length !== dimension) {
-                throw new Error(`Vector dimension mismatch for id=${id}: got ${p.vector.length}, expected ${dimension}`);
+        for (let i = 0; i < points.length; i += UPSERT_BATCH_SIZE) {
+            const batch = points.slice(i, i + UPSERT_BATCH_SIZE);
+            let ddl;
+            let params;
+            if (CLIENT_SIDE_SERIALIZATION_ENABLED) {
+                ddl = `
+          DECLARE $rows AS List<Struct<
+            uid: Utf8,
+            point_id: Utf8,
+            embedding: String,
+            embedding_quantized: String,
+            payload: JsonDocument
+          >>;
+
+          UPSERT INTO ${tableName} (uid, point_id, embedding, embedding_quantized, payload)
+          SELECT
+            uid,
+            point_id,
+            embedding,
+            embedding_quantized,
+            payload
+          FROM AS_TABLE($rows);
+        `;
+                const rowType = Types.struct({
+                    uid: Types.UTF8,
+                    point_id: Types.UTF8,
+                    embedding: Types.BYTES,
+                    embedding_quantized: Types.BYTES,
+                    payload: Types.JSON_DOCUMENT,
+                });
+                const rowsValue = TypedValues.list(rowType, batch.map((p) => {
+                    const binaries = buildVectorBinaryParams(p.vector);
+                    return {
+                        uid,
+                        point_id: String(p.id),
+                        embedding: binaries.float,
+                        embedding_quantized: binaries.bit,
+                        payload: JSON.stringify(p.payload ?? {}),
+                    };
+                }));
+                params = {
+                    $rows: rowsValue,
+                };
             }
-            const ddl = `
-        DECLARE $uid AS Utf8;
-        DECLARE $id AS Utf8;
-        DECLARE $vec AS List<Float>;
-        DECLARE $payload AS JsonDocument;
-        UPSERT INTO ${tableName} (uid, point_id, embedding, embedding_bit, payload)
-        VALUES (
-          $uid,
-          $id,
-          Untag(Knn::ToBinaryStringFloat($vec), "FloatVector"),
-          Untag(Knn::ToBinaryStringBit($vec), "BitVector"),
-          $payload
-        );
-      `;
-            const params = {
-                $uid: TypedValues.utf8(uid),
-                $id: TypedValues.utf8(id),
-                $vec: buildVectorParam(p.vector),
-                $payload: buildJsonOrEmpty(p.payload),
-            };
+            else {
+                ddl = `
+          DECLARE $rows AS List<Struct<
+            uid: Utf8,
+            point_id: Utf8,
+            vec: List<Float>,
+            payload: JsonDocument
+          >>;
+
+          UPSERT INTO ${tableName} (uid, point_id, embedding, embedding_quantized, payload)
+          SELECT
+            uid,
+            point_id,
+            Untag(Knn::ToBinaryStringFloat(vec), "FloatVector") AS embedding,
+            Untag(Knn::ToBinaryStringBit(vec), "BitVector") AS embedding_quantized,
+            payload
+          FROM AS_TABLE($rows);
+        `;
+                const rowType = Types.struct({
+                    uid: Types.UTF8,
+                    point_id: Types.UTF8,
+                    vec: Types.list(Types.FLOAT),
+                    payload: Types.JSON_DOCUMENT,
+                });
+                const rowsValue = TypedValues.list(rowType, batch.map((p) => ({
+                    uid,
+                    point_id: String(p.id),
+                    vec: p.vector,
+                    payload: JSON.stringify(p.payload ?? {}),
+                })));
+                params = {
+                    $rows: rowsValue,
+                };
+            }
+            logger.debug({
+                tableName,
+                mode: CLIENT_SIDE_SERIALIZATION_ENABLED
+                    ? "one_table_upsert_client_side_serialization"
+                    : "one_table_upsert_server_side_knn",
+                batchSize: batch.length,
+                yql: ddl,
+                params: {
+                    rows: batch.map((p) => ({
+                        uid,
+                        point_id: String(p.id),
+                        vectorLength: p.vector.length,
+                        vectorPreview: p.vector.slice(0, 3),
+                        payload: p.payload ?? {},
+                    })),
+                },
+            }, "one_table upsert: executing YQL");
             await withRetry(() => s.executeQuery(ddl, params), {
                 isTransient: isTransientYdbError,
-                context: { tableName, id },
+                context: { tableName, batchSize: batch.length },
             });
-            upserted += 1;
+            upserted += batch.length;
         }
     });
     notifyUpsert(tableName, upserted);
     return upserted;
 }
-export async function searchPointsOneTable(tableName, queryVector, top, withPayload, distance, dimension, uid) {
+async function searchPointsOneTableExact(tableName, queryVector, top, withPayload, distance, dimension, uid) {
+    if (queryVector.length !== dimension) {
+        throw new Error(`Vector dimension mismatch: got ${queryVector.length}, expected ${dimension}`);
+    }
+    const { fn, order } = mapDistanceToKnnFn(distance);
+    const results = await withSession(async (s) => {
+        let yql;
+        let params;
+        if (CLIENT_SIDE_SERIALIZATION_ENABLED) {
+            const binaries = buildVectorBinaryParams(queryVector);
+            yql = `
+        DECLARE $qbinf AS String;
+        DECLARE $k AS Uint32;
+        DECLARE $uid AS Utf8;
+        SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
+        FROM ${tableName}
+        WHERE uid = $uid
+        ORDER BY score ${order}
+        LIMIT $k;
+      `;
+            params = {
+                $qbinf: typeof TypedValues.bytes === "function"
+                    ? TypedValues.bytes(binaries.float)
+                    : binaries.float,
+                $k: TypedValues.uint32(top),
+                $uid: TypedValues.utf8(uid),
+            };
+        }
+        else {
+            const qf = buildVectorParam(queryVector);
+            yql = `
+        DECLARE $qf AS List<Float>;
+        DECLARE $k AS Uint32;
+        DECLARE $uid AS Utf8;
+        $qbinf = Knn::ToBinaryStringFloat($qf);
+        SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
+        FROM ${tableName}
+        WHERE uid = $uid
+        ORDER BY score ${order}
+        LIMIT $k;
+      `;
+            params = {
+                $qf: qf,
+                $k: TypedValues.uint32(top),
+                $uid: TypedValues.utf8(uid),
+            };
+        }
+        logger.debug({
+            tableName,
+            distance,
+            top,
+            withPayload,
+            mode: CLIENT_SIDE_SERIALIZATION_ENABLED
+                ? "one_table_exact_client_side_serialization"
+                : "one_table_exact",
+            yql,
+            params: {
+                uid,
+                top,
+                vectorLength: queryVector.length,
+                vectorPreview: queryVector.slice(0, 3),
+            },
+        }, "one_table search (exact): executing YQL");
+        const rs = await s.executeQuery(yql, params);
+        const rowset = rs.resultSets?.[0];
+        const rows = (rowset?.rows ?? []);
+        return rows.map((row) => {
+            const id = row.items?.[0]?.textValue;
+            if (typeof id !== "string") {
+                throw new Error("point_id is missing in YDB search result");
+            }
+            let payload;
+            let scoreIdx = 1;
+            if (withPayload) {
+                const payloadText = row.items?.[1]?.textValue;
+                if (payloadText) {
+                    try {
+                        payload = JSON.parse(payloadText);
+                    }
+                    catch {
+                        payload = undefined;
+                    }
+                }
+                scoreIdx = 2;
+            }
+            const score = Number(row.items?.[scoreIdx]?.floatValue ?? row.items?.[scoreIdx]?.textValue);
+            return { id, score, ...(payload ? { payload } : {}) };
+        });
+    });
+    return results;
+}
+async function searchPointsOneTableApproximate(tableName, queryVector, top, withPayload, distance, dimension, uid, overfetchMultiplier) {
     if (queryVector.length !== dimension) {
         throw new Error(`Vector dimension mismatch: got ${queryVector.length}, expected ${dimension}`);
     }
     const { fn, order } = mapDistanceToKnnFn(distance);
     const { fn: bitFn, order: bitOrder } = mapDistanceToBitKnnFn(distance);
-    const qf = buildVectorParam(queryVector);
-    const candidateLimit = top * 10;
+    const safeTop = top > 0 ? top : 1;
+    const rawCandidateLimit = safeTop * overfetchMultiplier;
+    const candidateLimit = Math.max(safeTop, rawCandidateLimit);
     const results = await withSession(async (s) => {
-        // Phase 1: approximate candidate selection using embedding_bit
+        if (CLIENT_SIDE_SERIALIZATION_ENABLED) {
+            const binaries = buildVectorBinaryParams(queryVector);
+            // Phase 1: approximate candidate selection using embedding_quantized
+            const phase1Query = `
+        DECLARE $qbin_bit AS String;
+        DECLARE $k AS Uint32;
+        DECLARE $uid AS Utf8;
+        SELECT point_id
+        FROM ${tableName}
+        WHERE uid = $uid AND embedding_quantized IS NOT NULL
+        ORDER BY ${bitFn}(embedding_quantized, $qbin_bit) ${bitOrder}
+        LIMIT $k;
+      `;
+            logger.debug({
+                tableName,
+                distance,
+                top,
+                safeTop,
+                candidateLimit,
+                mode: "one_table_approximate_phase1_client_side_serialization",
+                yql: phase1Query,
+                params: {
+                    uid,
+                    top: candidateLimit,
+                    vectorLength: queryVector.length,
+                    vectorPreview: queryVector.slice(0, 3),
+                },
+            }, "one_table search (approximate, phase 1): executing YQL");
+            const phase1Params = {
+                $qbin_bit: typeof TypedValues.bytes === "function"
+                    ? TypedValues.bytes(binaries.bit)
+                    : binaries.bit,
+                $k: TypedValues.uint32(candidateLimit),
+                $uid: TypedValues.utf8(uid),
+            };
+            const rs1 = await s.executeQuery(phase1Query, phase1Params);
+            const rowset1 = rs1.resultSets?.[0];
+            const rows1 = (rowset1?.rows ?? []);
+            const candidateIds = rows1
+                .map((row) => row.items?.[0]?.textValue)
+                .filter((id) => typeof id === "string");
+            if (candidateIds.length === 0) {
+                return [];
+            }
+            // Phase 2: exact re-ranking on full-precision embedding for candidates only
+            const phase2Query = `
+        DECLARE $qbinf AS String;
+        DECLARE $k AS Uint32;
+        DECLARE $uid AS Utf8;
+        DECLARE $ids AS List<Utf8>;
+        SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
+        FROM ${tableName}
+        WHERE uid = $uid AND point_id IN $ids
+        ORDER BY score ${order}
+        LIMIT $k;
+      `;
+            logger.debug({
+                tableName,
+                distance,
+                top,
+                safeTop,
+                candidateCount: candidateIds.length,
+                mode: "one_table_approximate_phase2_client_side_serialization",
+                yql: phase2Query,
+                params: {
+                    uid,
+                    top: safeTop,
+                    vectorLength: queryVector.length,
+                    vectorPreview: queryVector.slice(0, 3),
+                    ids: candidateIds,
+                },
+            }, "one_table search (approximate, phase 2): executing YQL");
+            const idsParam = TypedValues.list(Types.UTF8, candidateIds);
+            const phase2Params = {
+                $qbinf: typeof TypedValues.bytes === "function"
+                    ? TypedValues.bytes(binaries.float)
+                    : binaries.float,
+                $k: TypedValues.uint32(safeTop),
+                $uid: TypedValues.utf8(uid),
+                $ids: idsParam,
+            };
+            const rs2 = await s.executeQuery(phase2Query, phase2Params);
+            const rowset2 = rs2.resultSets?.[0];
+            const rows2 = (rowset2?.rows ?? []);
+            return rows2.map((row) => {
+                const id = row.items?.[0]?.textValue;
+                if (typeof id !== "string") {
+                    throw new Error("point_id is missing in YDB search result");
+                }
+                let payload;
+                let scoreIdx = 1;
+                if (withPayload) {
+                    const payloadText = row.items?.[1]?.textValue;
+                    if (payloadText) {
+                        try {
+                            payload = JSON.parse(payloadText);
+                        }
+                        catch {
+                            payload = undefined;
+                        }
+                    }
+                    scoreIdx = 2;
+                }
+                const score = Number(row.items?.[scoreIdx]?.floatValue ?? row.items?.[scoreIdx]?.textValue);
+                return { id, score, ...(payload ? { payload } : {}) };
+            });
+        }
+        const qf = buildVectorParam(queryVector);
+        // Phase 1: approximate candidate selection using embedding_quantized
         const phase1Query = `
       DECLARE $qf AS List<Float>;
       DECLARE $k AS Uint32;
@@ -58,10 +347,25 @@ export async function searchPointsOneTable(tableName, queryVector, top, withPayl
       $qbin_bit = Knn::ToBinaryStringBit($qf);
       SELECT point_id
       FROM ${tableName}
-      WHERE uid = $uid AND embedding_bit IS NOT NULL
-      ORDER BY ${bitFn}(embedding_bit, $qbin_bit) ${bitOrder}
+      WHERE uid = $uid AND embedding_quantized IS NOT NULL
+      ORDER BY ${bitFn}(embedding_quantized, $qbin_bit) ${bitOrder}
       LIMIT $k;
     `;
+        logger.debug({
+            tableName,
+            distance,
+            top,
+            safeTop,
+            candidateLimit,
+            mode: "one_table_approximate_phase1",
+            yql: phase1Query,
+            params: {
+                uid,
+                top: candidateLimit,
+                vectorLength: queryVector.length,
+                vectorPreview: queryVector.slice(0, 3),
+            },
+        }, "one_table search (approximate, phase 1): executing YQL");
         const phase1Params = {
             $qf: qf,
             $k: TypedValues.uint32(candidateLimit),
@@ -89,10 +393,26 @@ export async function searchPointsOneTable(tableName, queryVector, top, withPayl
       ORDER BY score ${order}
       LIMIT $k;
     `;
+        logger.debug({
+            tableName,
+            distance,
+            top,
+            safeTop,
+            candidateCount: candidateIds.length,
+            mode: "one_table_approximate_phase2",
+            yql: phase2Query,
+            params: {
+                uid,
+                top: safeTop,
+                vectorLength: queryVector.length,
+                vectorPreview: queryVector.slice(0, 3),
+                ids: candidateIds,
+            },
+        }, "one_table search (approximate, phase 2): executing YQL");
         const idsParam = TypedValues.list(Types.UTF8, candidateIds);
         const phase2Params = {
             $qf: qf,
-            $k: TypedValues.uint32(top),
+            $k: TypedValues.uint32(safeTop),
             $uid: TypedValues.utf8(uid),
             $ids: idsParam,
         };
@@ -124,6 +444,12 @@ export async function searchPointsOneTable(tableName, queryVector, top, withPayl
     });
     return results;
 }
+export async function searchPointsOneTable(tableName, queryVector, top, withPayload, distance, dimension, uid, mode, overfetchMultiplier) {
+    if (mode === SearchMode.Exact) {
+        return await searchPointsOneTableExact(tableName, queryVector, top, withPayload, distance, dimension, uid);
+    }
+    return await searchPointsOneTableApproximate(tableName, queryVector, top, withPayload, distance, dimension, uid, overfetchMultiplier);
+}
 export async function deletePointsOneTable(tableName, ids, uid) {
     let deleted = 0;
     await withSession(async (s) => {

package/dist/utils/vectorBinary.d.ts

@@ -0,0 +1,2 @@
+export declare function vectorToFloatBinary(vector: number[]): Buffer;
+export declare function vectorToBitBinary(vector: number[]): Buffer;

package/dist/utils/vectorBinary.js

@@ -0,0 +1,35 @@
+export function vectorToFloatBinary(vector) {
+    if (vector.length === 0) {
+        return Buffer.from([1]);
+    }
+    const buffer = Buffer.alloc(vector.length * 4 + 1);
+    for (let i = 0; i < vector.length; i += 1) {
+        const value = vector[i];
+        if (!Number.isFinite(value)) {
+            throw new Error(`Non-finite value in vector at index ${i}: ${value}`);
+        }
+        buffer.writeFloatLE(value, i * 4);
+    }
+    buffer.writeUInt8(1, vector.length * 4);
+    return buffer;
+}
+export function vectorToBitBinary(vector) {
+    if (vector.length === 0) {
+        return Buffer.from([10]);
+    }
+    const byteCount = Math.ceil(vector.length / 8);
+    const buffer = Buffer.alloc(byteCount + 1);
+    for (let i = 0; i < vector.length; i += 1) {
+        const value = vector[i];
+        if (!Number.isFinite(value)) {
+            throw new Error(`Non-finite value in vector at index ${i}: ${value}`);
+        }
+        if (value > 0) {
+            const byteIndex = Math.floor(i / 8);
+            const bitIndex = i % 8;
+            buffer[byteIndex] |= 1 << bitIndex;
+        }
+    }
+    buffer.writeUInt8(10, byteCount);
+    return buffer;
+}

package/dist/ydb/helpers.d.ts

@@ -1,2 +1,6 @@
 export declare function buildVectorParam(vector: number[]): import("ydb-sdk-proto").Ydb.ITypedValue;
 export declare function buildJsonOrEmpty(payload?: Record<string, unknown>): import("ydb-sdk-proto").Ydb.ITypedValue;
+export declare function buildVectorBinaryParams(vector: number[]): {
+    float: Buffer<ArrayBufferLike>;
+    bit: Buffer<ArrayBufferLike>;
+};

package/dist/ydb/helpers.js

@@ -1,7 +1,14 @@
 import { Types, TypedValues } from "./client.js";
+import { vectorToFloatBinary, vectorToBitBinary, } from "../utils/vectorBinary.js";
 export function buildVectorParam(vector) {
     return TypedValues.list(Types.FLOAT, vector);
 }
 export function buildJsonOrEmpty(payload) {
     return TypedValues.jsonDocument(JSON.stringify(payload ?? {}));
 }
+export function buildVectorBinaryParams(vector) {
+    return {
+        float: vectorToFloatBinary(vector),
+        bit: vectorToBitBinary(vector),
+    };
+}

package/dist/ydb/schema.d.ts

@@ -1,3 +1,4 @@
 export declare const GLOBAL_POINTS_TABLE = "qdrant_all_points";
+export { UPSERT_BATCH_SIZE } from "../config/env.js";
 export declare function ensureMetaTable(): Promise<void>;
 export declare function ensureGlobalPointsTable(): Promise<void>;

package/dist/ydb/schema.js

@@ -2,6 +2,8 @@ import { withSession, TableDescription, Column, Types } from "./client.js";
 import { logger } from "../logging/logger.js";
 import { GLOBAL_POINTS_AUTOMIGRATE_ENABLED } from "../config/env.js";
 export const GLOBAL_POINTS_TABLE = "qdrant_all_points";
+// Shared YDB-related constants for repositories.
+export { UPSERT_BATCH_SIZE } from "../config/env.js";
 let globalPointsTableReady = false;
 function throwMigrationRequired(message) {
     logger.error(message);
@@ -39,59 +41,34 @@ export async function ensureGlobalPointsTable() {
             tableDescription = await s.describeTable(GLOBAL_POINTS_TABLE);
         }
         catch {
-            // Table doesn't exist, create it with all columns
+            // Table doesn't exist, create it with all columns using the new schema.
             const desc = new TableDescription()
-                .withColumns(new Column("uid", Types.UTF8), new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("embedding_bit", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
+                .withColumns(new Column("uid", Types.UTF8), new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("embedding_quantized", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
                 .withPrimaryKeys("uid", "point_id");
             await s.createTable(GLOBAL_POINTS_TABLE, desc);
             globalPointsTableReady = true;
             logger.info(`created global points table ${GLOBAL_POINTS_TABLE}`);
             return;
         }
-        // Table exists, check if embedding_bit column is present
+        // Table exists, require the new embedding_quantized column.
         const columns = tableDescription.columns ?? [];
-        const hasEmbeddingBit = columns.some((col) => col.name === "embedding_bit");
-        let needsBackfill = false;
-        if (!hasEmbeddingBit) {
+        const hasEmbeddingQuantized = columns.some((col) => col.name === "embedding_quantized");
+        if (!hasEmbeddingQuantized) {
             if (!GLOBAL_POINTS_AUTOMIGRATE_ENABLED) {
-                throwMigrationRequired(`Global points table ${GLOBAL_POINTS_TABLE} is missing required column embedding_bit; set YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE=true after backup to apply the migration manually.`);
+                throwMigrationRequired(`Global points table ${GLOBAL_POINTS_TABLE} is missing required column embedding_quantized; apply the migration (e.g., ALTER TABLE ${GLOBAL_POINTS_TABLE} RENAME COLUMN embedding_bit TO embedding_quantized) or set YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE=true after backup to allow automatic migration.`);
             }
             const alterDdl = `
           ALTER TABLE ${GLOBAL_POINTS_TABLE}
-          ADD COLUMN embedding_bit String;
+          ADD COLUMN embedding_quantized String;
         `;
             const rawSession = s;
             await rawSession.api.executeSchemeQuery({
                 sessionId: rawSession.sessionId,
                 yqlText: alterDdl,
             });
-            logger.info(`added embedding_bit column to existing table ${GLOBAL_POINTS_TABLE}`);
-            needsBackfill = true;
+            logger.info(`added embedding_quantized column to existing table ${GLOBAL_POINTS_TABLE}`);
         }
-        else {
-            const checkNullsDdl = `
-          SELECT 1 AS has_null
-          FROM ${GLOBAL_POINTS_TABLE}
-          WHERE embedding_bit IS NULL
-          LIMIT 1;
-        `;
-            const checkRes = await s.executeQuery(checkNullsDdl);
-            const rows = checkRes?.resultSets?.[0]?.rows ?? [];
-            needsBackfill = rows.length > 0;
-        }
-        if (needsBackfill) {
-            if (!GLOBAL_POINTS_AUTOMIGRATE_ENABLED) {
-                throwMigrationRequired(`Global points table ${GLOBAL_POINTS_TABLE} requires backfill for embedding_bit; set YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE=true after backup to apply the migration manually.`);
-            }
-            const backfillDdl = `
-          UPDATE ${GLOBAL_POINTS_TABLE}
-          SET embedding_bit = Untag(Knn::ToBinaryStringBit(Knn::FloatFromBinaryString(embedding)), "BitVector")
-          WHERE embedding_bit IS NULL;
-        `;
-            await s.executeQuery(backfillDdl);
-            logger.info(`backfilled embedding_bit column from embedding in ${GLOBAL_POINTS_TABLE}`);
-        }
-        // Mark table ready only after schema (and any required backfill) succeed
+        // Mark table ready after schema checks/migrations succeed.
         globalPointsTableReady = true;
     });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ydb-qdrant",
-  "version": "4.6.0",
+  "version": "4.7.0",
   "main": "dist/package/api.js",
   "types": "dist/package/api.d.ts",
   "exports": {
@@ -16,8 +16,10 @@
   "scripts": {
     "test": "vitest run --exclude \"test/integration/**\"",
     "test:coverage": "vitest run --coverage --exclude \"test/integration/**\"",
-    "test:integration": "VECTOR_INDEX_BUILD_ENABLED=false vitest run test/integration/YdbRealIntegration.index-disabled.test.ts && VECTOR_INDEX_BUILD_ENABLED=true vitest run test/integration/YdbRealIntegration.test.ts test/integration/YdbRecallIntegration.test.ts && YDB_QDRANT_COLLECTION_STORAGE_MODE=one_table vitest run test/integration/YdbRealIntegration.one-table.test.ts",
-    "test:recall": "VECTOR_INDEX_BUILD_ENABLED=true vitest run test/integration/YdbRecallIntegration.test.ts && YDB_QDRANT_COLLECTION_STORAGE_MODE=one_table vitest run test/integration/YdbRealIntegration.one-table.test.ts",
+    "test:integration": "VECTOR_INDEX_BUILD_ENABLED=false vitest run test/integration/YdbRealIntegration.index-disabled.test.ts && VECTOR_INDEX_BUILD_ENABLED=true vitest run test/integration/YdbRealIntegration.test.ts test/integration/YdbRecallIntegration.test.ts && YDB_QDRANT_COLLECTION_STORAGE_MODE=one_table YDB_QDRANT_SEARCH_MODE=approximate vitest run test/integration/YdbRealIntegration.one-table.test.ts && YDB_QDRANT_COLLECTION_STORAGE_MODE=one_table YDB_QDRANT_SEARCH_MODE=exact vitest run test/integration/YdbRealIntegration.one-table.test.ts",
+    "test:recall": "VECTOR_INDEX_BUILD_ENABLED=true vitest run test/integration/YdbRecallIntegration.test.ts && YDB_QDRANT_COLLECTION_STORAGE_MODE=one_table YDB_QDRANT_SEARCH_MODE=approximate vitest run test/integration/YdbRealIntegration.one-table.test.ts && YDB_QDRANT_COLLECTION_STORAGE_MODE=one_table YDB_QDRANT_SEARCH_MODE=exact vitest run test/integration/YdbRealIntegration.one-table.test.ts",
+    "load:soak": "k6 run loadtest/soak-test.js",
+    "load:stress": "k6 run loadtest/stress-test.js",
     "build": "tsc -p tsconfig.json",
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "dev": "tsx watch src/index.ts",
@@ -81,4 +83,4 @@
     "typescript-eslint": "^8.47.0",
     "vitest": "^4.0.12"
   }
-}
+}