ydb-qdrant 3.0.0 → 4.1.0

package/dist/services/QdrantService.js

@@ -1,313 +0,0 @@
-import { sanitizeCollectionName, sanitizeTenantId, metaKeyFor, tableNameFor, } from "../utils/tenant.js";
-import { CreateCollectionReq, DeletePointsReq, SearchReq, UpsertPointsReq, } from "../types.js";
-import { ensureMetaTable } from "../ydb/schema.js";
-import { createCollection as repoCreateCollection, deleteCollection as repoDeleteCollection, getCollectionMeta, } from "../repositories/collectionsRepo.js";
-import { deletePoints as repoDeletePoints, searchPoints as repoSearchPoints, upsertPoints as repoUpsertPoints, } from "../repositories/pointsRepo.js";
-import { requestIndexBuild } from "../indexing/IndexScheduler.js";
-import { logger } from "../logging/logger.js";
-import { VECTOR_INDEX_BUILD_ENABLED } from "../config/env.js";
-export class QdrantServiceError extends Error {
-    statusCode;
-    payload;
-    constructor(statusCode, payload, message) {
-        super(message ?? String(payload.error));
-        this.statusCode = statusCode;
-        this.payload = payload;
-    }
-}
-function normalizeCollectionContext(input) {
-    const tenant = sanitizeTenantId(input.tenant);
-    const collection = sanitizeCollectionName(input.collection);
-    const metaKey = metaKeyFor(tenant, collection);
-    return { tenant, collection, metaKey };
-}
-export async function putCollectionIndex(ctx) {
-    await ensureMetaTable();
-    const normalized = normalizeCollectionContext(ctx);
-    const meta = await getCollectionMeta(normalized.metaKey);
-    if (!meta) {
-        throw new QdrantServiceError(404, {
-            status: "error",
-            error: "collection not found",
-        });
-    }
-    return { acknowledged: true };
-}
-export async function createCollection(ctx, body) {
-    await ensureMetaTable();
-    const normalized = normalizeCollectionContext(ctx);
-    const parsed = CreateCollectionReq.safeParse(body);
-    if (!parsed.success) {
-        throw new QdrantServiceError(400, {
-            status: "error",
-            error: parsed.error.flatten(),
-        });
-    }
-    const dim = parsed.data.vectors.size;
-    const distance = parsed.data.vectors.distance;
-    const vectorType = parsed.data.vectors.data_type ?? "float";
-    const existing = await getCollectionMeta(normalized.metaKey);
-    if (existing) {
-        if (existing.dimension === dim &&
-            existing.distance === distance &&
-            existing.vectorType === vectorType) {
-            return { name: normalized.collection, tenant: normalized.tenant };
-        }
-        const errorMessage = `Collection already exists with different config: dimension=${existing.dimension}, distance=${existing.distance}, type=${existing.vectorType}`;
-        throw new QdrantServiceError(400, {
-            status: "error",
-            error: errorMessage,
-        });
-    }
-    const tableName = tableNameFor(normalized.tenant, normalized.collection);
-    await repoCreateCollection(normalized.metaKey, dim, distance, vectorType, tableName);
-    return { name: normalized.collection, tenant: normalized.tenant };
-}
-export async function getCollection(ctx) {
-    await ensureMetaTable();
-    const normalized = normalizeCollectionContext(ctx);
-    const meta = await getCollectionMeta(normalized.metaKey);
-    if (!meta) {
-        throw new QdrantServiceError(404, {
-            status: "error",
-            error: "collection not found",
-        });
-    }
-    return {
-        name: normalized.collection,
-        vectors: {
-            size: meta.dimension,
-            distance: meta.distance,
-            data_type: meta.vectorType,
-        },
-    };
-}
-export async function deleteCollection(ctx) {
-    await ensureMetaTable();
-    const normalized = normalizeCollectionContext(ctx);
-    await repoDeleteCollection(normalized.metaKey);
-    return { acknowledged: true };
-}
-let loggedIndexBuildDisabled = false;
-function isNumberArray(value) {
-    return Array.isArray(value) && value.every((x) => typeof x === "number");
-}
-function extractVectorLoose(body, depth = 0) {
-    if (!body || typeof body !== "object" || depth > 3) {
-        return undefined;
-    }
-    const obj = body;
-    if (isNumberArray(obj.vector))
-        return obj.vector;
-    if (isNumberArray(obj.embedding))
-        return obj.embedding;
-    const query = obj.query;
-    if (query) {
-        const queryVector = query["vector"];
-        if (isNumberArray(queryVector))
-            return queryVector;
-        const nearest = query["nearest"];
-        if (nearest && isNumberArray(nearest.vector)) {
-            return nearest.vector;
-        }
-    }
-    const nearest = obj.nearest;
-    if (nearest && isNumberArray(nearest.vector)) {
-        return nearest.vector;
-    }
-    for (const key of Object.keys(obj)) {
-        const value = obj[key];
-        if (isNumberArray(value)) {
-            return value;
-        }
-    }
-    for (const key of Object.keys(obj)) {
-        const value = obj[key];
-        if (value && typeof value === "object") {
-            const found = extractVectorLoose(value, depth + 1);
-            if (found) {
-                return found;
-            }
-        }
-    }
-    return undefined;
-}
-function normalizeSearchBodyForSearch(body) {
-    if (!body || typeof body !== "object") {
-        return {
-            vector: undefined,
-            top: undefined,
-            withPayload: undefined,
-            scoreThreshold: undefined,
-        };
-    }
-    const b = body;
-    const rawVector = b["vector"];
-    const vector = isNumberArray(rawVector) ? rawVector : undefined;
-    const rawTop = b["top"];
-    const rawLimit = b["limit"];
-    const topFromTop = typeof rawTop === "number" ? rawTop : undefined;
-    const topFromLimit = typeof rawLimit === "number" ? rawLimit : undefined;
-    const top = topFromTop ?? topFromLimit;
-    let withPayload;
-    const rawWithPayload = b["with_payload"];
-    if (typeof rawWithPayload === "boolean") {
-        withPayload = rawWithPayload;
-    }
-    else if (Array.isArray(rawWithPayload) ||
-        typeof rawWithPayload === "object") {
-        withPayload = true;
-    }
-    const thresholdRaw = b["score_threshold"];
-    const thresholdValue = typeof thresholdRaw === "number" ? thresholdRaw : Number(thresholdRaw);
-    const scoreThreshold = Number.isFinite(thresholdValue)
-        ? thresholdValue
-        : undefined;
-    return { vector, top, withPayload, scoreThreshold };
-}
-function normalizeSearchBodyForQuery(body) {
-    if (!body || typeof body !== "object") {
-        return {
-            vector: undefined,
-            top: undefined,
-            withPayload: undefined,
-            scoreThreshold: undefined,
-        };
-    }
-    const b = body;
-    const vector = extractVectorLoose(b);
-    const rawTop = b["top"];
-    const rawLimit = b["limit"];
-    const topFromTop = typeof rawTop === "number" ? rawTop : undefined;
-    const topFromLimit = typeof rawLimit === "number" ? rawLimit : undefined;
-    const top = topFromTop ?? topFromLimit;
-    let withPayload;
-    const rawWithPayload = b["with_payload"];
-    if (typeof rawWithPayload === "boolean") {
-        withPayload = rawWithPayload;
-    }
-    else if (Array.isArray(rawWithPayload) ||
-        typeof rawWithPayload === "object") {
-        withPayload = true;
-    }
-    const thresholdRaw = b["score_threshold"];
-    const thresholdValue = typeof thresholdRaw === "number" ? thresholdRaw : Number(thresholdRaw);
-    const scoreThreshold = Number.isFinite(thresholdValue)
-        ? thresholdValue
-        : undefined;
-    return { vector, top, withPayload, scoreThreshold };
-}
-export async function upsertPoints(ctx, body) {
-    await ensureMetaTable();
-    const normalized = normalizeCollectionContext(ctx);
-    const meta = await getCollectionMeta(normalized.metaKey);
-    if (!meta) {
-        throw new QdrantServiceError(404, {
-            status: "error",
-            error: "collection not found",
-        });
-    }
-    const parsed = UpsertPointsReq.safeParse(body);
-    if (!parsed.success) {
-        throw new QdrantServiceError(400, {
-            status: "error",
-            error: parsed.error.flatten(),
-        });
-    }
-    const upserted = await repoUpsertPoints(meta.table, parsed.data.points, meta.dimension);
-    if (VECTOR_INDEX_BUILD_ENABLED) {
-        requestIndexBuild(meta.table, meta.dimension, meta.distance, meta.vectorType);
-    }
-    else if (!loggedIndexBuildDisabled) {
-        logger.info({ table: meta.table }, "vector index building disabled by env; skipping automatic emb_idx rebuilds");
-        loggedIndexBuildDisabled = true;
-    }
-    return { upserted };
-}
-async function executeSearch(ctx, normalizedSearch, source) {
-    await ensureMetaTable();
-    const normalized = normalizeCollectionContext(ctx);
-    logger.info({ tenant: normalized.tenant, collection: normalized.collection }, `${source}: resolve collection meta`);
-    const meta = await getCollectionMeta(normalized.metaKey);
-    if (!meta) {
-        logger.warn({
-            tenant: normalized.tenant,
-            collection: normalized.collection,
-            metaKey: normalized.metaKey,
-        }, `${source}: collection not found`);
-        throw new QdrantServiceError(404, {
-            status: "error",
-            error: "collection not found",
-        });
-    }
-    const parsed = SearchReq.safeParse({
-        vector: normalizedSearch.vector,
-        top: normalizedSearch.top,
-        with_payload: normalizedSearch.withPayload,
-    });
-    if (!parsed.success) {
-        logger.warn({
-            tenant: normalized.tenant,
-            collection: normalized.collection,
-            issues: parsed.error.issues,
-        }, `${source}: invalid payload`);
-        throw new QdrantServiceError(400, {
-            status: "error",
-            error: parsed.error.flatten(),
-        });
-    }
-    logger.info({
-        tenant: normalized.tenant,
-        collection: normalized.collection,
-        top: parsed.data.top,
-        queryVectorLen: parsed.data.vector.length,
-        collectionDim: meta.dimension,
-        distance: meta.distance,
-        vectorType: meta.vectorType,
-    }, `${source}: executing`);
-    const hits = await repoSearchPoints(meta.table, parsed.data.vector, parsed.data.top, parsed.data.with_payload, meta.distance, meta.dimension);
-    const threshold = normalizedSearch.scoreThreshold;
-    const filtered = threshold === undefined
-        ? hits
-        : hits.filter((hit) => {
-            const isSimilarity = meta.distance === "Cosine" || meta.distance === "Dot";
-            if (isSimilarity) {
-                return hit.score >= threshold;
-            }
-            return hit.score <= threshold;
-        });
-    logger.info({
-        tenant: normalized.tenant,
-        collection: normalized.collection,
-        hits: hits.length,
-    }, `${source}: completed`);
-    return { points: filtered };
-}
-export async function searchPoints(ctx, body) {
-    const normalizedSearch = normalizeSearchBodyForSearch(body);
-    return await executeSearch(ctx, normalizedSearch, "search");
-}
-export async function queryPoints(ctx, body) {
-    const normalizedSearch = normalizeSearchBodyForQuery(body);
-    return await executeSearch(ctx, normalizedSearch, "query");
-}
-export async function deletePoints(ctx, body) {
-    await ensureMetaTable();
-    const normalized = normalizeCollectionContext(ctx);
-    const meta = await getCollectionMeta(normalized.metaKey);
-    if (!meta) {
-        throw new QdrantServiceError(404, {
-            status: "error",
-            error: "collection not found",
-        });
-    }
-    const parsed = DeletePointsReq.safeParse(body);
-    if (!parsed.success) {
-        throw new QdrantServiceError(400, {
-            status: "error",
-            error: parsed.error.flatten(),
-        });
-    }
-    const deleted = await repoDeletePoints(meta.table, parsed.data.points);
-    return { deleted };
-}