ydb-qdrant 3.0.0 → 4.1.0

package/dist/services/PointsService.js

@@ -0,0 +1,128 @@
+import { UpsertPointsReq, SearchReq, DeletePointsReq } from "../types.js";
+import { ensureMetaTable } from "../ydb/schema.js";
+import { getCollectionMeta } from "../repositories/collectionsRepo.js";
+import { deletePoints as repoDeletePoints, searchPoints as repoSearchPoints, upsertPoints as repoUpsertPoints, } from "../repositories/pointsRepo.js";
+import { requestIndexBuild } from "../indexing/IndexScheduler.js";
+import { logger } from "../logging/logger.js";
+import { VECTOR_INDEX_BUILD_ENABLED } from "../config/env.js";
+import { QdrantServiceError } from "./errors.js";
+import { normalizeCollectionContext, resolvePointsTableAndUid, } from "./CollectionService.js";
+import { normalizeSearchBodyForSearch, normalizeSearchBodyForQuery, } from "../utils/normalization.js";
+let loggedIndexBuildDisabled = false;
+export async function upsertPoints(ctx, body) {
+    await ensureMetaTable();
+    const normalized = normalizeCollectionContext(ctx);
+    const meta = await getCollectionMeta(normalized.metaKey);
+    if (!meta) {
+        throw new QdrantServiceError(404, {
+            status: "error",
+            error: "collection not found",
+        });
+    }
+    const parsed = UpsertPointsReq.safeParse(body);
+    if (!parsed.success) {
+        throw new QdrantServiceError(400, {
+            status: "error",
+            error: parsed.error.flatten(),
+        });
+    }
+    const { tableName, uid } = await resolvePointsTableAndUid(normalized, meta);
+    const upserted = await repoUpsertPoints(tableName, parsed.data.points, meta.dimension, uid);
+    if (VECTOR_INDEX_BUILD_ENABLED) {
+        requestIndexBuild(tableName, meta.dimension, meta.distance, meta.vectorType);
+    }
+    else if (!loggedIndexBuildDisabled) {
+        logger.info({ table: tableName }, "vector index building disabled by env; skipping automatic emb_idx rebuilds");
+        loggedIndexBuildDisabled = true;
+    }
+    return { upserted };
+}
+async function executeSearch(ctx, normalizedSearch, source) {
+    await ensureMetaTable();
+    const normalized = normalizeCollectionContext(ctx);
+    logger.info({ tenant: normalized.tenant, collection: normalized.collection }, `${source}: resolve collection meta`);
+    const meta = await getCollectionMeta(normalized.metaKey);
+    if (!meta) {
+        logger.warn({
+            tenant: normalized.tenant,
+            collection: normalized.collection,
+            metaKey: normalized.metaKey,
+        }, `${source}: collection not found`);
+        throw new QdrantServiceError(404, {
+            status: "error",
+            error: "collection not found",
+        });
+    }
+    const parsed = SearchReq.safeParse({
+        vector: normalizedSearch.vector,
+        top: normalizedSearch.top,
+        with_payload: normalizedSearch.withPayload,
+    });
+    if (!parsed.success) {
+        logger.warn({
+            tenant: normalized.tenant,
+            collection: normalized.collection,
+            issues: parsed.error.issues,
+        }, `${source}: invalid payload`);
+        throw new QdrantServiceError(400, {
+            status: "error",
+            error: parsed.error.flatten(),
+        });
+    }
+    const { tableName, uid } = await resolvePointsTableAndUid(normalized, meta);
+    logger.info({
+        tenant: normalized.tenant,
+        collection: normalized.collection,
+        top: parsed.data.top,
+        queryVectorLen: parsed.data.vector.length,
+        collectionDim: meta.dimension,
+        distance: meta.distance,
+        vectorType: meta.vectorType,
+    }, `${source}: executing`);
+    const hits = await repoSearchPoints(tableName, parsed.data.vector, parsed.data.top, parsed.data.with_payload, meta.distance, meta.dimension, uid);
+    const threshold = normalizedSearch.scoreThreshold;
+    const filtered = threshold === undefined
+        ? hits
+        : hits.filter((hit) => {
+            const isSimilarity = meta.distance === "Cosine" || meta.distance === "Dot";
+            if (isSimilarity) {
+                return hit.score >= threshold;
+            }
+            return hit.score <= threshold;
+        });
+    logger.info({
+        tenant: normalized.tenant,
+        collection: normalized.collection,
+        hits: hits.length,
+    }, `${source}: completed`);
+    return { points: filtered };
+}
+export async function searchPoints(ctx, body) {
+    const normalizedSearch = normalizeSearchBodyForSearch(body);
+    return await executeSearch(ctx, normalizedSearch, "search");
+}
+export async function queryPoints(ctx, body) {
+    const normalizedSearch = normalizeSearchBodyForQuery(body);
+    return await executeSearch(ctx, normalizedSearch, "query");
+}
+export async function deletePoints(ctx, body) {
+    await ensureMetaTable();
+    const normalized = normalizeCollectionContext(ctx);
+    const meta = await getCollectionMeta(normalized.metaKey);
+    if (!meta) {
+        throw new QdrantServiceError(404, {
+            status: "error",
+            error: "collection not found",
+        });
+    }
+    const parsed = DeletePointsReq.safeParse(body);
+    if (!parsed.success) {
+        throw new QdrantServiceError(400, {
+            status: "error",
+            error: parsed.error.flatten(),
+        });
+    }
+    const { tableName, uid } = await resolvePointsTableAndUid(normalized, meta);
+    const deleted = await repoDeletePoints(tableName, parsed.data.points, uid);
+    return { deleted };
+}

package/dist/services/errors.d.ts

@@ -0,0 +1,9 @@
+export interface QdrantServiceErrorPayload {
+    status: "error";
+    error: unknown;
+}
+export declare class QdrantServiceError extends Error {
+    readonly statusCode: number;
+    readonly payload: QdrantServiceErrorPayload;
+    constructor(statusCode: number, payload: QdrantServiceErrorPayload, message?: string);
+}

package/dist/services/errors.js

@@ -0,0 +1,9 @@
+export class QdrantServiceError extends Error {
+    statusCode;
+    payload;
+    constructor(statusCode, payload, message) {
+        super(message ?? String(payload.error));
+        this.statusCode = statusCode;
+        this.payload = payload;
+    }
+}

package/dist/utils/distance.d.ts

@@ -0,0 +1,6 @@
+import type { DistanceKind } from "../types.js";
+export declare function mapDistanceToKnnFn(distance: DistanceKind): {
+    fn: string;
+    order: "ASC" | "DESC";
+};
+export declare function mapDistanceToIndexParam(distance: DistanceKind): string;

package/dist/utils/distance.js

@@ -0,0 +1,28 @@
+export function mapDistanceToKnnFn(distance) {
+    switch (distance) {
+        case "Cosine":
+            return { fn: "Knn::CosineSimilarity", order: "DESC" };
+        case "Dot":
+            return { fn: "Knn::InnerProductSimilarity", order: "DESC" };
+        case "Euclid":
+            return { fn: "Knn::EuclideanDistance", order: "ASC" };
+        case "Manhattan":
+            return { fn: "Knn::ManhattanDistance", order: "ASC" };
+        default:
+            return { fn: "Knn::CosineSimilarity", order: "DESC" };
+    }
+}
+export function mapDistanceToIndexParam(distance) {
+    switch (distance) {
+        case "Cosine":
+            return "cosine";
+        case "Dot":
+            return "inner_product";
+        case "Euclid":
+            return "euclidean";
+        case "Manhattan":
+            return "manhattan";
+        default:
+            return "cosine";
+    }
+}

package/dist/utils/normalization.d.ts

@@ -0,0 +1,10 @@
+export interface SearchNormalizationResult {
+    vector: number[] | undefined;
+    top: number | undefined;
+    withPayload: boolean | undefined;
+    scoreThreshold: number | undefined;
+}
+export declare function isNumberArray(value: unknown): value is number[];
+export declare function extractVectorLoose(body: unknown, depth?: number): number[] | undefined;
+export declare function normalizeSearchBodyForSearch(body: unknown): SearchNormalizationResult;
+export declare function normalizeSearchBodyForQuery(body: unknown): SearchNormalizationResult;

package/dist/utils/normalization.js

@@ -0,0 +1,92 @@
+export function isNumberArray(value) {
+    return Array.isArray(value) && value.every((x) => typeof x === "number");
+}
+export function extractVectorLoose(body, depth = 0) {
+    if (!body || typeof body !== "object" || depth > 3) {
+        return undefined;
+    }
+    const obj = body;
+    if (isNumberArray(obj.vector))
+        return obj.vector;
+    if (isNumberArray(obj.embedding))
+        return obj.embedding;
+    const query = obj.query;
+    if (query) {
+        const queryVector = query["vector"];
+        if (isNumberArray(queryVector))
+            return queryVector;
+        const nearest = query["nearest"];
+        if (nearest && isNumberArray(nearest.vector)) {
+            return nearest.vector;
+        }
+    }
+    const nearest = obj.nearest;
+    if (nearest && isNumberArray(nearest.vector)) {
+        return nearest.vector;
+    }
+    for (const key of Object.keys(obj)) {
+        const value = obj[key];
+        if (isNumberArray(value)) {
+            return value;
+        }
+    }
+    for (const key of Object.keys(obj)) {
+        const value = obj[key];
+        if (value && typeof value === "object") {
+            const found = extractVectorLoose(value, depth + 1);
+            if (found) {
+                return found;
+            }
+        }
+    }
+    return undefined;
+}
+export function normalizeSearchBodyForSearch(body) {
+    if (!body || typeof body !== "object") {
+        return {
+            vector: undefined,
+            top: undefined,
+            withPayload: undefined,
+            scoreThreshold: undefined,
+        };
+    }
+    const b = body;
+    const rawVector = b["vector"];
+    const vector = isNumberArray(rawVector) ? rawVector : undefined;
+    return normalizeSearchCommon(b, vector);
+}
+export function normalizeSearchBodyForQuery(body) {
+    if (!body || typeof body !== "object") {
+        return {
+            vector: undefined,
+            top: undefined,
+            withPayload: undefined,
+            scoreThreshold: undefined,
+        };
+    }
+    const b = body;
+    const vector = extractVectorLoose(b);
+    return normalizeSearchCommon(b, vector);
+}
+function normalizeSearchCommon(b, vector) {
+    const rawTop = b["top"];
+    const rawLimit = b["limit"];
+    const topFromTop = typeof rawTop === "number" ? rawTop : undefined;
+    const topFromLimit = typeof rawLimit === "number" ? rawLimit : undefined;
+    const top = topFromTop ?? topFromLimit;
+    let withPayload;
+    const rawWithPayload = b["with_payload"];
+    if (typeof rawWithPayload === "boolean") {
+        withPayload = rawWithPayload;
+    }
+    else if (Array.isArray(rawWithPayload) ||
+        typeof rawWithPayload === "object") {
+        withPayload = true;
+    }
+    const thresholdRaw = b["score_threshold"];
+    const thresholdValue = typeof thresholdRaw === "number" ? thresholdRaw : Number(thresholdRaw);
+    const scoreThreshold = Number.isFinite(thresholdValue)
+        ? thresholdValue
+        : undefined;
+    return { vector, top, withPayload, scoreThreshold };
+}

package/dist/utils/retry.d.ts

@@ -0,0 +1,8 @@
+export interface RetryOptions {
+    maxRetries?: number;
+    baseDelayMs?: number;
+    isTransient?: (error: unknown) => boolean;
+    context?: Record<string, unknown>;
+}
+export declare function isTransientYdbError(error: unknown): boolean;
+export declare function withRetry<T>(fn: () => Promise<T>, options?: RetryOptions): Promise<T>;

package/dist/utils/retry.js

@@ -0,0 +1,28 @@
+import { logger } from "../logging/logger.js";
+const DEFAULT_MAX_RETRIES = 6;
+const DEFAULT_BASE_DELAY_MS = 250;
+export function isTransientYdbError(error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    return /Aborted|schema version mismatch|Table metadata loading|Failed to load metadata/i.test(msg);
+}
+export async function withRetry(fn, options = {}) {
+    const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
+    const baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;
+    const isTransient = options.isTransient ?? isTransientYdbError;
+    const context = options.context ?? {};
+    let attempt = 0;
+    while (true) {
+        try {
+            return await fn();
+        }
+        catch (e) {
+            if (!isTransient(e) || attempt >= maxRetries) {
+                throw e;
+            }
+            const backoffMs = Math.floor(baseDelayMs * Math.pow(2, attempt) + Math.random() * 100);
+            logger.warn({ ...context, attempt, backoffMs }, "operation aborted due to transient error; retrying");
+            await new Promise((r) => setTimeout(r, backoffMs));
+            attempt += 1;
+        }
+    }
+}

package/dist/utils/tenant.d.ts

@@ -2,3 +2,4 @@ export declare function sanitizeCollectionName(name: string): string;
 export declare function sanitizeTenantId(tenantId: string | undefined): string;
 export declare function tableNameFor(tenantId: string, collection: string): string;
 export declare function metaKeyFor(tenantId: string, collection: string): string;
+export declare function uidFor(tenantId: string, collectionName: string): string;

package/dist/utils/tenant.js

@@ -15,3 +15,6 @@ export function tableNameFor(tenantId, collection) {
 export function metaKeyFor(tenantId, collection) {
     return `${sanitizeTenantId(tenantId)}/${sanitizeCollectionName(collection)}`;
 }
+export function uidFor(tenantId, collectionName) {
+    return tableNameFor(tenantId, collectionName);
+}

package/dist/ydb/schema.d.ts

@@ -1 +1,3 @@
+export declare const GLOBAL_POINTS_TABLE = "qdrant_all_points";
 export declare function ensureMetaTable(): Promise<void>;
+export declare function ensureGlobalPointsTable(): Promise<void>;

package/dist/ydb/schema.js

@@ -1,5 +1,7 @@
 import { withSession, TableDescription, Column, Types } from "./client.js";
 import { logger } from "../logging/logger.js";
+export const GLOBAL_POINTS_TABLE = "qdrant_all_points";
+let globalPointsTableReady = false;
 export async function ensureMetaTable() {
     try {
         await withSession(async (s) => {
@@ -22,3 +24,28 @@ export async function ensureMetaTable() {
         logger.debug({ err }, "ensureMetaTable: ignored");
     }
 }
+export async function ensureGlobalPointsTable() {
+    if (globalPointsTableReady) {
+        return;
+    }
+    try {
+        await withSession(async (s) => {
+            try {
+                await s.describeTable(GLOBAL_POINTS_TABLE);
+                globalPointsTableReady = true;
+                return;
+            }
+            catch {
+                const desc = new TableDescription()
+                    .withColumns(new Column("uid", Types.UTF8), new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
+                    .withPrimaryKeys("uid", "point_id");
+                await s.createTable(GLOBAL_POINTS_TABLE, desc);
+                globalPointsTableReady = true;
+                logger.info(`created global points table ${GLOBAL_POINTS_TABLE}`);
+            }
+        });
+    }
+    catch (err) {
+        logger.debug({ err }, "ensureGlobalPointsTable: ignored");
+    }
+}

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "ydb-qdrant",
-  "version": "3.0.0",
-  "main": "dist/package/Api.js",
-  "types": "dist/package/Api.d.ts",
+  "version": "4.1.0",
+  "main": "dist/package/api.js",
+  "types": "dist/package/api.d.ts",
   "exports": {
-    ".": "./dist/package/Api.js",
+    ".": "./dist/package/api.js",
     "./server": "./dist/server.js"
   },
   "files": [

package/dist/package/Api.js

@@ -1,79 +0,0 @@
-import { readyOrThrow, configureDriver } from "../ydb/client.js";
-import { ensureMetaTable } from "../ydb/schema.js";
-import { createCollection as serviceCreateCollection, deleteCollection as serviceDeleteCollection, getCollection as serviceGetCollection, putCollectionIndex as servicePutCollectionIndex, upsertPoints as serviceUpsertPoints, searchPoints as serviceSearchPoints, deletePoints as serviceDeletePoints, } from "../services/QdrantService.js";
-export { QdrantServiceError } from "../services/QdrantService.js";
-export { CreateCollectionReq, UpsertPointsReq, SearchReq, DeletePointsReq, } from "../types.js";
-export async function createYdbQdrantClient(options = {}) {
-    if (options.endpoint !== undefined ||
-        options.database !== undefined ||
-        options.connectionString !== undefined ||
-        options.authService !== undefined) {
-        configureDriver({
-            endpoint: options.endpoint,
-            database: options.database,
-            connectionString: options.connectionString,
-            authService: options.authService,
-        });
-    }
-    await readyOrThrow();
-    await ensureMetaTable();
-    const defaultTenant = options.defaultTenant ?? "default";
-    const resolveTenant = (tenant) => tenant ?? defaultTenant;
-    const client = {
-        async createCollection(collection, body) {
-            const tenant = resolveTenant(undefined);
-            return await serviceCreateCollection({ tenant, collection }, body);
-        },
-        async getCollection(collection) {
-            const tenant = resolveTenant(undefined);
-            return await serviceGetCollection({ tenant, collection });
-        },
-        async deleteCollection(collection) {
-            const tenant = resolveTenant(undefined);
-            return await serviceDeleteCollection({ tenant, collection });
-        },
-        async putCollectionIndex(collection) {
-            const tenant = resolveTenant(undefined);
-            return await servicePutCollectionIndex({ tenant, collection });
-        },
-        async upsertPoints(collection, body) {
-            const tenant = resolveTenant(undefined);
-            return await serviceUpsertPoints({ tenant, collection }, body);
-        },
-        async searchPoints(collection, body) {
-            const tenant = resolveTenant(undefined);
-            return await serviceSearchPoints({ tenant, collection }, body);
-        },
-        async deletePoints(collection, body) {
-            const tenant = resolveTenant(undefined);
-            return await serviceDeletePoints({ tenant, collection }, body);
-        },
-        forTenant(tenantId) {
-            const tenant = tenantId;
-            return {
-                createCollection(collection, body) {
-                    return serviceCreateCollection({ tenant, collection }, body);
-                },
-                getCollection(collection) {
-                    return serviceGetCollection({ tenant, collection });
-                },
-                deleteCollection(collection) {
-                    return serviceDeleteCollection({ tenant, collection });
-                },
-                putCollectionIndex(collection) {
-                    return servicePutCollectionIndex({ tenant, collection });
-                },
-                upsertPoints(collection, body) {
-                    return serviceUpsertPoints({ tenant, collection }, body);
-                },
-                searchPoints(collection, body) {
-                    return serviceSearchPoints({ tenant, collection }, body);
-                },
-                deletePoints(collection, body) {
-                    return serviceDeletePoints({ tenant, collection }, body);
-                },
-            };
-        },
-    };
-    return client;
-}

package/dist/services/QdrantService.d.ts

@@ -1,54 +0,0 @@
-import { type DistanceKind } from "../types.js";
-export interface QdrantServiceErrorPayload {
-    status: "error";
-    error: unknown;
-}
-export declare class QdrantServiceError extends Error {
-    readonly statusCode: number;
-    readonly payload: QdrantServiceErrorPayload;
-    constructor(statusCode: number, payload: QdrantServiceErrorPayload, message?: string);
-}
-interface CollectionContextInput {
-    tenant: string | undefined;
-    collection: string;
-}
-export declare function putCollectionIndex(ctx: CollectionContextInput): Promise<{
-    acknowledged: boolean;
-}>;
-export declare function createCollection(ctx: CollectionContextInput, body: unknown): Promise<{
-    name: string;
-    tenant: string;
-}>;
-export declare function getCollection(ctx: CollectionContextInput): Promise<{
-    name: string;
-    vectors: {
-        size: number;
-        distance: DistanceKind;
-        data_type: string;
-    };
-}>;
-export declare function deleteCollection(ctx: CollectionContextInput): Promise<{
-    acknowledged: boolean;
-}>;
-type PointsContextInput = CollectionContextInput;
-export declare function upsertPoints(ctx: PointsContextInput, body: unknown): Promise<{
-    upserted: number;
-}>;
-export declare function searchPoints(ctx: PointsContextInput, body: unknown): Promise<{
-    points: Array<{
-        id: string;
-        score: number;
-        payload?: Record<string, unknown>;
-    }>;
-}>;
-export declare function queryPoints(ctx: PointsContextInput, body: unknown): Promise<{
-    points: Array<{
-        id: string;
-        score: number;
-        payload?: Record<string, unknown>;
-    }>;
-}>;
-export declare function deletePoints(ctx: PointsContextInput, body: unknown): Promise<{
-    deleted: number;
-}>;
-export {};