ydb-qdrant 2.0.0

package/dist/services/QdrantService.js

@@ -0,0 +1,268 @@
+import { sanitizeCollectionName, sanitizeTenantId, metaKeyFor, tableNameFor, } from "../utils/tenant.js";
+import { CreateCollectionReq, DeletePointsReq, SearchReq, UpsertPointsReq, } from "../types.js";
+import { ensureMetaTable } from "../ydb/schema.js";
+import { createCollection as repoCreateCollection, deleteCollection as repoDeleteCollection, getCollectionMeta, } from "../repositories/collectionsRepo.js";
+import { deletePoints as repoDeletePoints, searchPoints as repoSearchPoints, upsertPoints as repoUpsertPoints, } from "../repositories/pointsRepo.js";
+import { requestIndexBuild } from "../indexing/IndexScheduler.js";
+import { logger } from "../logging/logger.js";
+export class QdrantServiceError extends Error {
+    statusCode;
+    payload;
+    constructor(statusCode, payload, message) {
+        super(message ?? String(payload.error));
+        this.statusCode = statusCode;
+        this.payload = payload;
+    }
+}
+function normalizeCollectionContext(input) {
+    const tenant = sanitizeTenantId(input.tenant);
+    const collection = sanitizeCollectionName(input.collection);
+    const metaKey = metaKeyFor(tenant, collection);
+    return { tenant, collection, metaKey };
+}
+export async function putCollectionIndex(ctx) {
+    await ensureMetaTable();
+    const normalized = normalizeCollectionContext(ctx);
+    const meta = await getCollectionMeta(normalized.metaKey);
+    if (!meta) {
+        throw new QdrantServiceError(404, {
+            status: "error",
+            error: "collection not found",
+        });
+    }
+    return { acknowledged: true };
+}
+export async function createCollection(ctx, body) {
+    await ensureMetaTable();
+    const normalized = normalizeCollectionContext(ctx);
+    const parsed = CreateCollectionReq.safeParse(body);
+    if (!parsed.success) {
+        throw new QdrantServiceError(400, {
+            status: "error",
+            error: parsed.error.flatten(),
+        });
+    }
+    const dim = parsed.data.vectors.size;
+    const distance = parsed.data.vectors.distance;
+    const vectorType = parsed.data.vectors.data_type ?? "float";
+    const existing = await getCollectionMeta(normalized.metaKey);
+    if (existing) {
+        if (existing.dimension === dim &&
+            existing.distance === distance &&
+            existing.vectorType === vectorType) {
+            return { name: normalized.collection, tenant: normalized.tenant };
+        }
+        const errorMessage = `Collection already exists with different config: dimension=${existing.dimension}, distance=${existing.distance}, type=${existing.vectorType}`;
+        throw new QdrantServiceError(400, {
+            status: "error",
+            error: errorMessage,
+        });
+    }
+    const tableName = tableNameFor(normalized.tenant, normalized.collection);
+    await repoCreateCollection(normalized.metaKey, dim, distance, vectorType, tableName);
+    return { name: normalized.collection, tenant: normalized.tenant };
+}
+export async function getCollection(ctx) {
+    await ensureMetaTable();
+    const normalized = normalizeCollectionContext(ctx);
+    const meta = await getCollectionMeta(normalized.metaKey);
+    if (!meta) {
+        throw new QdrantServiceError(404, {
+            status: "error",
+            error: "collection not found",
+        });
+    }
+    return {
+        name: normalized.collection,
+        vectors: {
+            size: meta.dimension,
+            distance: meta.distance,
+            data_type: meta.vectorType,
+        },
+    };
+}
+export async function deleteCollection(ctx) {
+    await ensureMetaTable();
+    const normalized = normalizeCollectionContext(ctx);
+    await repoDeleteCollection(normalized.metaKey);
+    return { acknowledged: true };
+}
+function isNumberArray(value) {
+    return Array.isArray(value) && value.every((x) => typeof x === "number");
+}
+function extractVectorLoose(body, depth = 0) {
+    if (!body || typeof body !== "object" || depth > 3) {
+        return undefined;
+    }
+    const obj = body;
+    if (isNumberArray(obj.vector))
+        return obj.vector;
+    if (isNumberArray(obj.embedding))
+        return obj.embedding;
+    const query = obj.query;
+    if (query) {
+        const queryVector = query.vector;
+        if (isNumberArray(queryVector))
+            return queryVector;
+        const nearest = query.nearest;
+        if (nearest && isNumberArray(nearest.vector)) {
+            return nearest.vector;
+        }
+    }
+    const nearest = obj.nearest;
+    if (nearest && isNumberArray(nearest.vector)) {
+        return nearest.vector;
+    }
+    for (const key of Object.keys(obj)) {
+        const value = obj[key];
+        if (isNumberArray(value)) {
+            return value;
+        }
+    }
+    for (const key of Object.keys(obj)) {
+        const value = obj[key];
+        if (value && typeof value === "object") {
+            const found = extractVectorLoose(value, depth + 1);
+            if (found) {
+                return found;
+            }
+        }
+    }
+    return undefined;
+}
+function normalizeSearchBodyForSearch(body) {
+    const b = body;
+    const vector = Array.isArray(b?.vector) ? b.vector : undefined;
+    const topFromTop = typeof b?.top === "number" ? b.top : undefined;
+    const topFromLimit = typeof b?.limit === "number" ? b.limit : undefined;
+    const top = topFromTop ?? topFromLimit;
+    let withPayload;
+    const rawWithPayload = b?.with_payload;
+    if (typeof rawWithPayload === "boolean") {
+        withPayload = rawWithPayload;
+    }
+    else if (Array.isArray(rawWithPayload) || typeof rawWithPayload === "object") {
+        withPayload = true;
+    }
+    const thresholdValue = Number(b?.score_threshold);
+    const scoreThreshold = Number.isFinite(thresholdValue) ? thresholdValue : undefined;
+    return { vector, top, withPayload, scoreThreshold };
+}
+function normalizeSearchBodyForQuery(body) {
+    const b = body;
+    const vector = extractVectorLoose(b);
+    const topFromTop = typeof b?.top === "number" ? b.top : undefined;
+    const topFromLimit = typeof b?.limit === "number" ? b.limit : undefined;
+    const top = topFromTop ?? topFromLimit;
+    let withPayload;
+    const rawWithPayload = b?.with_payload;
+    if (typeof rawWithPayload === "boolean") {
+        withPayload = rawWithPayload;
+    }
+    else if (Array.isArray(rawWithPayload) || typeof rawWithPayload === "object") {
+        withPayload = true;
+    }
+    const thresholdValue = Number(b?.score_threshold);
+    const scoreThreshold = Number.isFinite(thresholdValue) ? thresholdValue : undefined;
+    return { vector, top, withPayload, scoreThreshold };
+}
+export async function upsertPoints(ctx, body) {
+    await ensureMetaTable();
+    const normalized = normalizeCollectionContext(ctx);
+    const meta = await getCollectionMeta(normalized.metaKey);
+    if (!meta) {
+        throw new QdrantServiceError(404, {
+            status: "error",
+            error: "collection not found",
+        });
+    }
+    const parsed = UpsertPointsReq.safeParse(body);
+    if (!parsed.success) {
+        throw new QdrantServiceError(400, {
+            status: "error",
+            error: parsed.error.flatten(),
+        });
+    }
+    const upserted = await repoUpsertPoints(meta.table, parsed.data.points, meta.vectorType, meta.dimension);
+    requestIndexBuild(meta.table, meta.dimension, meta.distance, meta.vectorType);
+    return { upserted };
+}
+async function executeSearch(ctx, normalizedSearch, source) {
+    await ensureMetaTable();
+    const normalized = normalizeCollectionContext(ctx);
+    logger.info({ tenant: normalized.tenant, collection: normalized.collection }, `${source}: resolve collection meta`);
+    const meta = await getCollectionMeta(normalized.metaKey);
+    if (!meta) {
+        logger.warn({ tenant: normalized.tenant, collection: normalized.collection, metaKey: normalized.metaKey }, `${source}: collection not found`);
+        throw new QdrantServiceError(404, {
+            status: "error",
+            error: "collection not found",
+        });
+    }
+    const parsed = SearchReq.safeParse({
+        vector: normalizedSearch.vector,
+        top: normalizedSearch.top,
+        with_payload: normalizedSearch.withPayload,
+    });
+    if (!parsed.success) {
+        logger.warn({
+            tenant: normalized.tenant,
+            collection: normalized.collection,
+            issues: parsed.error.issues,
+        }, `${source}: invalid payload`);
+        throw new QdrantServiceError(400, {
+            status: "error",
+            error: parsed.error.flatten(),
+        });
+    }
+    logger.info({
+        tenant: normalized.tenant,
+        collection: normalized.collection,
+        top: parsed.data.top,
+        queryVectorLen: parsed.data.vector.length,
+        collectionDim: meta.dimension,
+        distance: meta.distance,
+        vectorType: meta.vectorType,
+    }, `${source}: executing`);
+    const hits = await repoSearchPoints(meta.table, parsed.data.vector, parsed.data.top, parsed.data.with_payload, meta.distance, meta.vectorType, meta.dimension);
+    const threshold = normalizedSearch.scoreThreshold;
+    const filtered = threshold === undefined
+        ? hits
+        : hits.filter((hit) => {
+            const isSimilarity = meta.distance === "Cosine" || meta.distance === "Dot";
+            if (isSimilarity) {
+                return hit.score >= threshold;
+            }
+            return hit.score <= threshold;
+        });
+    logger.info({ tenant: normalized.tenant, collection: normalized.collection, hits: hits.length }, `${source}: completed`);
+    return { points: filtered };
+}
+export async function searchPoints(ctx, body) {
+    const normalizedSearch = normalizeSearchBodyForSearch(body);
+    return await executeSearch(ctx, normalizedSearch, "search");
+}
+export async function queryPoints(ctx, body) {
+    const normalizedSearch = normalizeSearchBodyForQuery(body);
+    return await executeSearch(ctx, normalizedSearch, "query");
+}
+export async function deletePoints(ctx, body) {
+    await ensureMetaTable();
+    const normalized = normalizeCollectionContext(ctx);
+    const meta = await getCollectionMeta(normalized.metaKey);
+    if (!meta) {
+        throw new QdrantServiceError(404, {
+            status: "error",
+            error: "collection not found",
+        });
+    }
+    const parsed = DeletePointsReq.safeParse(body);
+    if (!parsed.success) {
+        throw new QdrantServiceError(400, {
+            status: "error",
+            error: parsed.error.flatten(),
+        });
+    }
+    const deleted = await repoDeletePoints(meta.table, parsed.data.points);
+    return { deleted };
+}

package/dist/types.d.ts

@@ -0,0 +1,28 @@
+import { z } from "zod";
+export type DistanceKind = "Cosine" | "Euclid" | "Dot" | "Manhattan";
+export type VectorType = "float" | "uint8";
+export declare const CreateCollectionReq: z.ZodObject<{
+    vectors: z.ZodObject<{
+        size: z.ZodNumber;
+        distance: z.ZodType<DistanceKind>;
+        data_type: z.ZodOptional<z.ZodEnum<{
+            float: "float";
+            uint8: "uint8";
+        }>>;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+export declare const UpsertPointsReq: z.ZodObject<{
+    points: z.ZodArray<z.ZodObject<{
+        id: z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>;
+        vector: z.ZodArray<z.ZodNumber>;
+        payload: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare const SearchReq: z.ZodObject<{
+    vector: z.ZodArray<z.ZodNumber>;
+    top: z.ZodNumber;
+    with_payload: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strip>;
+export declare const DeletePointsReq: z.ZodObject<{
+    points: z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>>;
+}, z.core.$strip>;

package/dist/types.js

@@ -0,0 +1,30 @@
+import { z } from "zod";
+export const CreateCollectionReq = z.object({
+    vectors: z.object({
+        size: z.number().int().positive(),
+        distance: z.enum([
+            "Cosine",
+            "Euclid",
+            "Dot",
+            "Manhattan",
+        ]),
+        data_type: z.enum(["float", "uint8"]).optional(),
+    }),
+});
+export const UpsertPointsReq = z.object({
+    points: z
+        .array(z.object({
+        id: z.union([z.string(), z.number()]),
+        vector: z.array(z.number()),
+        payload: z.record(z.string(), z.any()).optional(),
+    }))
+        .min(1),
+});
+export const SearchReq = z.object({
+    vector: z.array(z.number()).min(1),
+    top: z.number().int().positive().max(1000),
+    with_payload: z.boolean().optional(),
+});
+export const DeletePointsReq = z.object({
+    points: z.array(z.union([z.string(), z.number()])).min(1),
+});

package/dist/utils/tenant.d.ts

@@ -0,0 +1,4 @@
+export declare function sanitizeCollectionName(name: string): string;
+export declare function sanitizeTenantId(tenantId: string | undefined): string;
+export declare function tableNameFor(tenantId: string, collection: string): string;
+export declare function metaKeyFor(tenantId: string, collection: string): string;

package/dist/utils/tenant.js

@@ -0,0 +1,17 @@
+export function sanitizeCollectionName(name) {
+    const cleaned = name.replace(/[^a-zA-Z0-9_]/g, "_").replace(/_+/g, "_");
+    const lowered = cleaned.toLowerCase().replace(/^_+/, "");
+    return lowered.length > 0 ? lowered : "collection";
+}
+export function sanitizeTenantId(tenantId) {
+    const raw = (tenantId ?? "default").toString();
+    const cleaned = raw.replace(/[^a-zA-Z0-9_]/g, "_").replace(/_+/g, "_");
+    const lowered = cleaned.toLowerCase().replace(/^_+/, "");
+    return lowered.length > 0 ? lowered : "default";
+}
+export function tableNameFor(tenantId, collection) {
+    return `qdr_${sanitizeTenantId(tenantId)}__${sanitizeCollectionName(collection)}`;
+}
+export function metaKeyFor(tenantId, collection) {
+    return `${sanitizeTenantId(tenantId)}/${sanitizeCollectionName(collection)}`;
+}

package/dist/ydb/client.d.ts

@@ -0,0 +1,6 @@
+import type { Session } from "ydb-sdk";
+declare const Types: any, TypedValues: any, TableDescription: any, Column: any;
+export { Types, TypedValues, TableDescription, Column };
+export declare const driver: any;
+export declare function readyOrThrow(): Promise<void>;
+export declare function withSession<T>(fn: (s: Session) => Promise<T>): Promise<T>;

package/dist/ydb/client.js

@@ -0,0 +1,20 @@
+import { createRequire } from "module";
+import { YDB_DATABASE, YDB_ENDPOINT } from "../config/env.js";
+const require = createRequire(import.meta.url);
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { Driver, getCredentialsFromEnv, Types, TypedValues, TableDescription, Column, } = require("ydb-sdk");
+export { Types, TypedValues, TableDescription, Column };
+export const driver = new Driver({
+    endpoint: YDB_ENDPOINT,
+    database: YDB_DATABASE,
+    authService: getCredentialsFromEnv(),
+});
+export async function readyOrThrow() {
+    const ok = await driver.ready(10000);
+    if (!ok) {
+        throw new Error("YDB driver is not ready in 10s. Check connectivity and credentials.");
+    }
+}
+export async function withSession(fn) {
+    return await driver.tableClient.withSession(fn, 15000);
+}

package/dist/ydb/helpers.d.ts

@@ -0,0 +1,2 @@
+export declare function buildVectorParam(vector: number[], vectorType: "float" | "uint8"): any;
+export declare function buildJsonOrEmpty(payload?: Record<string, unknown>): any;

package/dist/ydb/helpers.js

@@ -0,0 +1,45 @@
+import { Types, TypedValues } from "./client.js";
+export function buildVectorParam(vector, vectorType) {
+    let list;
+    if (vectorType === "uint8") {
+        // Check if vector is already quantized (integers in [0,255])
+        const isAlreadyQuantized = vector.every(v => Number.isInteger(v) && v >= 0 && v <= 255);
+        if (isAlreadyQuantized) {
+            list = vector;
+        }
+        else {
+            // Float embeddings need quantization. Per YDB docs (knn.md lines 282-294):
+            // Formula: ((x - min) / (max - min)) * 255
+            const min = Math.min(...vector);
+            const max = Math.max(...vector);
+            // Determine quantization strategy based on detected range
+            if (min >= 0 && max <= 1.01) {
+                // Normalized [0,1] embeddings (common for some models)
+                list = vector.map(v => Math.round(Math.max(0, Math.min(1, v)) * 255));
+            }
+            else if (min >= -1.01 && max <= 1.01) {
+                // Normalized [-1,1] embeddings (most common)
+                // Map to [0,255]: ((x + 1) / 2) * 255 = (x + 1) * 127.5
+                list = vector.map(v => Math.round((Math.max(-1, Math.min(1, v)) + 1) * 127.5));
+            }
+            else {
+                // General case: linear scaling from [min,max] to [0,255]
+                const range = max - min;
+                if (range > 0) {
+                    list = vector.map(v => Math.round(((v - min) / range) * 255));
+                }
+                else {
+                    // All values identical; map to midpoint
+                    list = vector.map(() => 127);
+                }
+            }
+        }
+    }
+    else {
+        list = vector;
+    }
+    return TypedValues.list(vectorType === "uint8" ? Types.UINT8 : Types.FLOAT, list);
+}
+export function buildJsonOrEmpty(payload) {
+    return TypedValues.jsonDocument(JSON.stringify(payload ?? {}));
+}

package/dist/ydb/schema.d.ts

@@ -0,0 +1 @@
+export declare function ensureMetaTable(): Promise<void>;

package/dist/ydb/schema.js

@@ -0,0 +1,24 @@
+import { withSession, TableDescription, Column, Types } from "./client.js";
+import { logger } from "../logging/logger.js";
+export async function ensureMetaTable() {
+    try {
+        await withSession(async (s) => {
+            // If table exists, describeTable will succeed
+            try {
+                await s.describeTable("qdr__collections");
+                return;
+            }
+            catch {
+                // create via schema API
+                const desc = new TableDescription()
+                    .withColumns(new Column("collection", Types.UTF8), new Column("table_name", Types.UTF8), new Column("vector_dimension", Types.UINT32), new Column("distance", Types.UTF8), new Column("vector_type", Types.UTF8), new Column("created_at", Types.TIMESTAMP))
+                    .withPrimaryKey("collection");
+                await s.createTable("qdr__collections", desc);
+                logger.info("created metadata table qdr__collections");
+            }
+        });
+    }
+    catch (err) {
+        logger.debug({ err }, "ensureMetaTable: ignored");
+    }
+}

package/package.json

@@ -0,0 +1,73 @@
+{
+  "name": "ydb-qdrant",
+  "version": "2.0.0",
+  "main": "dist/Api.js",
+  "types": "dist/Api.d.ts",
+  "exports": {
+    ".": "./dist/Api.js",
+    "./server": "./dist/server.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "test": "vitest run",
+    "build": "tsc -p tsconfig.json",
+    "dev": "tsx watch src/index.ts",
+    "start": "node --experimental-specifier-resolution=node --enable-source-maps dist/index.js",
+    "smoke": "npm run build && node --experimental-specifier-resolution=node --enable-source-maps dist/SmokeTest.js",
+    "lint": "biome check .",
+    "prepublishOnly": "npm test && npm run build"
+  },
+  "keywords": [
+    "ydb",
+    "vector-search",
+    "approximate-nearest-neighbor",
+    "qdrant-compatible",
+    "embeddings",
+    "nodejs",
+    "typescript",
+    "express",
+    "yql",
+    "grpc",
+    "yandex-cloud",
+    "ann",
+    "semantic-search",
+    "rag",
+    "pino",
+    "zod"
+  ],
+  "author": "",
+  "license": "ISC",
+  "description": "Qdrant-compatible Node.js/TypeScript API that stores/searches embeddings in YDB using approximate coarse-to-fine vector search (quantized uint8 preselect + float refine).",
+  "type": "module",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@bufbuild/protobuf": "^2.10.0",
+    "@grpc/grpc-js": "^1.14.0",
+    "@ydbjs/api": "^6.0.4",
+    "@ydbjs/core": "^6.0.4",
+    "@ydbjs/query": "^6.0.4",
+    "@ydbjs/value": "^6.0.4",
+    "dotenv": "^17.2.3",
+    "express": "^5.1.0",
+    "nice-grpc": "^2.1.13",
+    "pino": "^10.1.0",
+    "ydb-sdk": "^5.11.1",
+    "zod": "^4.1.12"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^2.2.7",
+    "@types/express": "^5.0.3",
+    "@types/node": "^24.9.1",
+    "docsify-cli": "^4.4.4",
+    "node-plantuml-latest": "^2.4.0",
+    "tsx": "^4.20.6",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.12"
+  }
+}