ydb-qdrant 8.1.0 → 9.0.3

package/dist/repositories/pointsRepo.one-table/Upsert.js

@@ -1,9 +1,14 @@
-import { TypedValues, Types, withSession, Ydb as YdbRuntime, createBulkUpsertSettingsWithTimeout, } from "../../ydb/client.js";
+import { TypedValues, Types, withSession, Ydb as YdbRuntime, createBulkUpsertSettingsWithTimeout, createExecuteQuerySettingsWithTimeout, } from "../../ydb/client.js";
 import { buildVectorBinaryParams } from "../../ydb/helpers.js";
-import { withRetry, isTransientYdbError } from "../../utils/retry.js";
-import { UPSERT_BATCH_SIZE } from "../../ydb/schema.js";
+import { withRetry, isTransientYdbErrorInAcquiredSession, } from "../../utils/retry.js";
+import { POINTS_BY_FILE_LOOKUP_TABLE, UPSERT_BATCH_SIZE, ensurePointsByFileTable, } from "../../ydb/schema.js";
 import { UPSERT_OPERATION_TIMEOUT_MS } from "../../config/env.js";
 import { logger } from "../../logging/logger.js";
+import { elapsedMsSince, getMonotonicTimeNs, } from "../../logging/requestContext.js";
+import { computePayloadSign } from "../../utils/PayloadSign.js";
+import { extractPathPrefix } from "../../utils/pathPrefix.js";
+import { expandPathPrefixes } from "../../utils/prefixExpansion.js";
+import { isComputePoolEnabled, isComputePoolQueueAtLimitError, runPrepareUpsertBatch, } from "../../compute/ComputePool.js";
 function assertPointVectorsDimension(args) {
     for (const p of args.points) {
         const id = String(p.id);
@@ -12,7 +17,7 @@ function assertPointVectorsDimension(args) {
             const vectorPreview = previewLength > 0 ? p.vector.slice(0, previewLength) : [];
             logger.warn({
                 tableName: args.tableName,
-                uid: args.uid,
+                collection: args.collection,
                 pointId: id,
                 vectorLen: p.vector.length,
                 expectedDimension: args.dimension,
@@ -22,64 +27,255 @@ function assertPointVectorsDimension(args) {
         }
     }
 }
-function buildBulkUpsertRowsValue(args) {
-    const rowType = Types.struct({
-        uid: Types.UTF8,
+function createBulkUpsertRowType() {
+    return Types.struct({
+        collection: Types.UTF8,
         point_id: Types.UTF8,
         embedding: Types.BYTES,
-        embedding_quantized: Types.BYTES,
         payload: Types.JSON_DOCUMENT,
+        payload_sign: Types.UTF8,
+        path_prefix: Types.optional(Types.UTF8),
+    });
+}
+function asBytesBuffer(value) {
+    // In worker_threads structured clone, Buffer arrives as Uint8Array.
+    // Wrap it into Buffer without copying again.
+    return Buffer.isBuffer(value)
+        ? value
+        : Buffer.from(value.buffer, value.byteOffset, value.byteLength);
+}
+function createPointsByFileLookupRowType() {
+    return Types.struct({
+        collection: Types.UTF8,
+        file_path: Types.UTF8,
+        point_id: Types.UTF8,
+    });
+}
+function buildPointsByFileLookupRows(args) {
+    return args.batch.flatMap((point) => {
+        const payloadObj = point.payload ?? {};
+        const filePath = extractPathPrefix(payloadObj);
+        if (!filePath) {
+            return [];
+        }
+        return expandPathPrefixes(filePath).map((prefix) => ({
+            collection: args.collection,
+            file_path: prefix,
+            point_id: String(point.id),
+        }));
     });
-    return TypedValues.list(rowType, args.batch.map((p) => {
+}
+async function syncPointsByFileLookupRows(args) {
+    const rows = buildPointsByFileLookupRows(args);
+    if (rows.length === 0) {
+        return;
+    }
+    const rowsValue = TypedValues.list(createPointsByFileLookupRowType(), rows);
+    const upsertLookupYql = `
+        DECLARE $rows AS List<Struct<
+            collection: Utf8,
+            file_path: Utf8,
+            point_id: Utf8
+        >>;
+
+        UPSERT INTO ${POINTS_BY_FILE_LOOKUP_TABLE}
+        SELECT collection, file_path, point_id FROM AS_TABLE($rows);
+    `;
+    await withRetry(async () => {
+        const settings = createExecuteQuerySettingsWithTimeout({
+            timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
+        });
+        await args.session.executeQuery(upsertLookupYql, {
+            $rows: rowsValue,
+        }, undefined, settings);
+    }, {
+        isTransient: isTransientYdbErrorInAcquiredSession,
+        context: {
+            operation: "syncPointsByFileLookupRows",
+            collection: args.collection,
+            batchSize: args.batch.length,
+            lookupRowsCount: rows.length,
+        },
+    });
+}
+function buildBulkUpsertRowsValue(args) {
+    const apiKey = args.apiKey.trim();
+    return TypedValues.list(createBulkUpsertRowType(), args.batch.map((p) => {
         const binaries = buildVectorBinaryParams(p.vector);
-        return {
-            uid: args.uid,
+        const payloadObj = p.payload ?? {};
+        const row = {
+            collection: args.collection,
             point_id: String(p.id),
             embedding: binaries.float,
-            embedding_quantized: binaries.bit,
-            payload: JSON.stringify(p.payload ?? {}),
+            payload: JSON.stringify(payloadObj),
+            payload_sign: computePayloadSign({ apiKey, payload: payloadObj }),
+            path_prefix: extractPathPrefix(payloadObj),
         };
+        return row;
     }));
 }
-export async function upsertPointsOneTable(tableName, points, dimension, uid) {
+async function buildBulkUpsertRowsValueWithWorkers(args) {
+    const prepared = await runPrepareUpsertBatch({
+        collection: args.collection,
+        apiKey: args.apiKey,
+        batch: args.batch,
+    });
+    return TypedValues.list(createBulkUpsertRowType(), prepared.map((row) => {
+        const mapped = {
+            collection: row.collection,
+            point_id: row.point_id,
+            embedding: asBytesBuffer(row.embedding),
+            payload: row.payload,
+            payload_sign: row.payload_sign,
+            path_prefix: row.path_prefix,
+        };
+        return mapped;
+    }));
+}
+export async function upsertPointsOneTable(tableName, points, dimension, collection, apiKey) {
     if (!tableName) {
         throw new Error("bulkUpsert: tableName is empty");
     }
-    assertPointVectorsDimension({ tableName, uid, points, dimension });
-    let upserted = 0;
-    await withSession(async (s) => {
-        const bulkSettings = createBulkUpsertSettingsWithTimeout({
-            timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
-        });
-        for (let i = 0; i < points.length; i += UPSERT_BATCH_SIZE) {
-            const batch = points.slice(i, i + UPSERT_BATCH_SIZE);
-            const rowsValue = buildBulkUpsertRowsValue({
-                uid,
-                batch,
+    assertPointVectorsDimension({ tableName, collection, points, dimension });
+    await ensurePointsByFileTable();
+    const repoStartNs = getMonotonicTimeNs();
+    let batchCount = 0;
+    let currentBatchIndex = 0;
+    let prepareRowsMsTotal = 0;
+    let lookupSyncMsTotal = 0;
+    let bulkUpsertMsTotal = 0;
+    let usedWorkersAny = false;
+    try {
+        await withSession(async (s) => {
+            const bulkSettings = createBulkUpsertSettingsWithTimeout({
+                timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
             });
-            if (logger.isLevelEnabled("debug")) {
-                logger.debug({
-                    tableName,
-                    mode: "one_table_bulk_upsert_client_side_serialization",
-                    batchSize: batch.length,
-                    params: {
-                        rows: batch.map((p) => ({
-                            uid,
-                            point_id: String(p.id),
-                            vectorLength: p.vector.length,
-                            vectorPreview: p.vector.slice(0, 3),
-                            payload: p.payload ?? {},
-                        })),
+            for (let i = 0; i < points.length; i += UPSERT_BATCH_SIZE) {
+                const batch = points.slice(i, i + UPSERT_BATCH_SIZE);
+                currentBatchIndex = Math.floor(i / UPSERT_BATCH_SIZE) + 1;
+                const batchStartNs = getMonotonicTimeNs();
+                const workersEnabled = isComputePoolEnabled();
+                let usedWorkers = false;
+                let rowsValue;
+                const prepareRowsStartNs = getMonotonicTimeNs();
+                if (workersEnabled) {
+                    try {
+                        usedWorkers = true;
+                        rowsValue = await buildBulkUpsertRowsValueWithWorkers({
+                            collection,
+                            batch,
+                            apiKey,
+                        });
+                    }
+                    catch (err) {
+                        if (!isComputePoolQueueAtLimitError(err)) {
+                            throw err;
+                        }
+                        // Backpressure: fall back to inline compute to avoid failing the request.
+                        usedWorkers = false;
+                        rowsValue = buildBulkUpsertRowsValue({
+                            collection,
+                            batch,
+                            apiKey,
+                        });
+                    }
+                }
+                else {
+                    rowsValue = buildBulkUpsertRowsValue({
+                        collection,
+                        batch,
+                        apiKey,
+                    });
+                }
+                const prepareRowsMs = elapsedMsSince(prepareRowsStartNs);
+                prepareRowsMsTotal += prepareRowsMs;
+                usedWorkersAny = usedWorkersAny || usedWorkers;
+                if (logger.isLevelEnabled("debug")) {
+                    logger.debug({
+                        tableName,
+                        mode: usedWorkers
+                            ? "one_table_bulk_upsert_worker_serialization"
+                            : "one_table_bulk_upsert_client_side_serialization",
+                        batchSize: batch.length,
+                        params: {
+                            rows: batch.map((p) => ({
+                                collection,
+                                point_id: String(p.id),
+                                vectorLength: p.vector.length,
+                                vectorPreview: p.vector.slice(0, 3),
+                                payload: p.payload ?? {},
+                            })),
+                        },
+                    }, "one_table upsert: executing BulkUpsert");
+                }
+                const lookupSyncStartNs = getMonotonicTimeNs();
+                await syncPointsByFileLookupRows({
+                    session: s,
+                    collection,
+                    batch,
+                });
+                const lookupSyncMs = elapsedMsSince(lookupSyncStartNs);
+                lookupSyncMsTotal += lookupSyncMs;
+                const typedRows = YdbRuntime.TypedValue.create(rowsValue);
+                const bulkUpsertStartNs = getMonotonicTimeNs();
+                await withRetry(() => s.bulkUpsert(tableName, typedRows, bulkSettings), {
+                    isTransient: isTransientYdbErrorInAcquiredSession,
+                    context: {
+                        tableName,
+                        batchSize: batch.length,
+                        mode: "bulkUpsert",
                     },
-                }, "one_table upsert: executing BulkUpsert");
+                });
+                const bulkUpsertMs = elapsedMsSince(bulkUpsertStartNs);
+                bulkUpsertMsTotal += bulkUpsertMs;
+                batchCount = currentBatchIndex;
+                if (logger.isLevelEnabled("debug")) {
+                    logger.debug({
+                        phase: "upsertRepoBatch",
+                        tableName,
+                        collection,
+                        batchIndex: currentBatchIndex,
+                        batchSize: batch.length,
+                        prepareRowsMs,
+                        lookupSyncMs,
+                        bulkUpsertMs,
+                        batchTotalMs: elapsedMsSince(batchStartNs),
+                        usedWorkers,
+                    }, "upsert: repo batch");
+                }
             }
-            const typedRows = YdbRuntime.TypedValue.create(rowsValue);
-            await withRetry(() => s.bulkUpsert(tableName, typedRows, bulkSettings), {
-                isTransient: isTransientYdbError,
-                context: { tableName, batchSize: batch.length, mode: "bulkUpsert" },
-            });
-            upserted += batch.length;
-        }
-    });
-    return upserted;
+        });
+    }
+    catch (err) {
+        logger.error({
+            err: err instanceof Error ? err : new Error(String(err)),
+            phase: "upsertRepo",
+            tableName,
+            collection,
+            repoTotalMs: elapsedMsSince(repoStartNs),
+            batchCount,
+            failedBatchIndex: currentBatchIndex > batchCount ? currentBatchIndex : undefined,
+            pointCount: points.length,
+            prepareRowsMsTotal,
+            lookupSyncMsTotal,
+            bulkUpsertMsTotal,
+            usedWorkersAny,
+        }, "upsert: failed");
+        throw err;
+    }
+    logger.info({
+        phase: "upsertRepo",
+        tableName,
+        collection,
+        repoTotalMs: elapsedMsSince(repoStartNs),
+        batchCount,
+        pointCount: points.length,
+        prepareRowsMsTotal,
+        lookupSyncMsTotal,
+        bulkUpsertMsTotal,
+        usedWorkersAny,
+    }, "upsert: repo summary");
+    // BulkUpsert is idempotent for the same (collection, point_id) keys; we report the
+    // number of points accepted by the API rather than rows actually changed.
+    return points.length;
 }

package/dist/repositories/pointsRepo.one-table.d.ts

@@ -1,3 +1,4 @@
 export { searchPointsOneTable } from "./pointsRepo.one-table/Search.js";
 export { upsertPointsOneTable } from "./pointsRepo.one-table/Upsert.js";
 export { deletePointsOneTable, deletePointsByPathSegmentsOneTable, } from "./pointsRepo.one-table/Delete.js";
+export { retrievePointsByIdsOneTable } from "./pointsRepo.one-table/Retrieve.js";

package/dist/repositories/pointsRepo.one-table.js

@@ -1,3 +1,4 @@
 export { searchPointsOneTable } from "./pointsRepo.one-table/Search.js";
 export { upsertPointsOneTable } from "./pointsRepo.one-table/Upsert.js";
 export { deletePointsOneTable, deletePointsByPathSegmentsOneTable, } from "./pointsRepo.one-table/Delete.js";
+export { retrievePointsByIdsOneTable } from "./pointsRepo.one-table/Retrieve.js";

package/dist/routes/collections.js

@@ -2,39 +2,55 @@ import { Router } from "express";
 import { putCollectionIndex, createCollection, getCollection, deleteCollection, } from "../services/CollectionService.js";
 import { QdrantServiceError } from "../services/errors.js";
 import { logger } from "../logging/logger.js";
+import { qdrantResponse } from "../utils/qdrantResponse.js";
+import { isAnonymousIdentityError, resolveRequestNamespaceUserUid, resolveRequestSigningKey, } from "../utils/requestIdentity.js";
 export const collectionsRouter = Router();
+function buildCollectionContext(req) {
+    const userUid = resolveRequestNamespaceUserUid(req);
+    return {
+        userUid,
+        collection: String(req.params.collection),
+        apiKey: resolveRequestSigningKey(req, userUid),
+        userAgent: req.header("User-Agent") ?? undefined,
+    };
+}
+function sendKnownRouteError(res, err) {
+    if (err instanceof QdrantServiceError) {
+        res.status(err.statusCode).json(err.payload);
+        return true;
+    }
+    if (isAnonymousIdentityError(err)) {
+        res.status(400).json({ status: "error", error: err.message });
+        return true;
+    }
+    return false;
+}
+// Fix #8: PUT index → UpdateResult shape
 collectionsRouter.put("/:collection/index", async (req, res) => {
+    const start = process.hrtime();
     try {
-        const result = await putCollectionIndex({
-            tenant: req.header("X-Tenant-Id") ?? undefined,
-            collection: String(req.params.collection),
-            apiKey: req.header("api-key") ?? undefined,
-            userAgent: req.header("User-Agent") ?? undefined,
-        });
-        res.json({ status: "ok", result });
+        await putCollectionIndex(buildCollectionContext(req));
+        res.json(qdrantResponse({ operation_id: 0, status: "completed" }, start));
     }
     catch (err) {
-        if (err instanceof QdrantServiceError) {
-            return res.status(err.statusCode).json(err.payload);
+        if (sendKnownRouteError(res, err)) {
+            return;
         }
         logger.error({ err }, "build index failed");
         const errorMessage = err instanceof Error ? err.message : String(err);
         res.status(500).json({ status: "error", error: errorMessage });
     }
 });
+// Fix #4: createCollection → result: true
 collectionsRouter.put("/:collection", async (req, res) => {
+    const start = process.hrtime();
     try {
-        const result = await createCollection({
-            tenant: req.header("X-Tenant-Id") ?? undefined,
-            collection: String(req.params.collection),
-            apiKey: req.header("api-key") ?? undefined,
-            userAgent: req.header("User-Agent") ?? undefined,
-        }, req.body);
-        res.json({ status: "ok", result });
+        await createCollection(buildCollectionContext(req), req.body);
+        res.json(qdrantResponse(true, start));
     }
     catch (err) {
-        if (err instanceof QdrantServiceError) {
-            return res.status(err.statusCode).json(err.payload);
+        if (sendKnownRouteError(res, err)) {
+            return;
         }
         logger.error({ err }, "create collection failed");
         const errorMessage = err instanceof Error ? err.message : String(err);
@@ -42,37 +58,30 @@ collectionsRouter.put("/:collection", async (req, res) => {
     }
 });
 collectionsRouter.get("/:collection", async (req, res) => {
+    const start = process.hrtime();
     try {
-        const result = await getCollection({
-            tenant: req.header("X-Tenant-Id") ?? undefined,
-            collection: String(req.params.collection),
-            apiKey: req.header("api-key") ?? undefined,
-            userAgent: req.header("User-Agent") ?? undefined,
-        });
-        res.json({ status: "ok", result });
+        const result = await getCollection(buildCollectionContext(req));
+        res.json(qdrantResponse(result, start));
     }
     catch (err) {
-        if (err instanceof QdrantServiceError) {
-            return res.status(err.statusCode).json(err.payload);
+        if (sendKnownRouteError(res, err)) {
+            return;
         }
         logger.error({ err }, "get collection failed");
         const errorMessage = err instanceof Error ? err.message : String(err);
         res.status(500).json({ status: "error", error: errorMessage });
     }
 });
+// Fix #5: deleteCollection → result: true
 collectionsRouter.delete("/:collection", async (req, res) => {
+    const start = process.hrtime();
     try {
-        const result = await deleteCollection({
-            tenant: req.header("X-Tenant-Id") ?? undefined,
-            collection: String(req.params.collection),
-            apiKey: req.header("api-key") ?? undefined,
-            userAgent: req.header("User-Agent") ?? undefined,
-        });
-        res.json({ status: "ok", result });
+        await deleteCollection(buildCollectionContext(req));
+        res.json(qdrantResponse(true, start));
     }
     catch (err) {
-        if (err instanceof QdrantServiceError) {
-            return res.status(err.statusCode).json(err.payload);
+        if (sendKnownRouteError(res, err)) {
+            return;
         }
         logger.error({ err }, "delete collection failed");
         const errorMessage = err instanceof Error ? err.message : String(err);