ydb-qdrant 8.1.1 → 9.0.3

package/dist/ydb/schema.js

@@ -1,12 +1,15 @@
-import { withSession, TableDescription, Column, Types, Ydb } from "./client.js";
+import { withSession, TableDescription, Column, Types, Ydb, } from "./client.js";
 import { logger } from "../logging/logger.js";
 export const GLOBAL_POINTS_TABLE = "qdrant_all_points";
+export const POINTS_BY_FILE_LOOKUP_TABLE = "qdrant_points_by_file";
 // Shared YDB-related constants for repositories.
 export { UPSERT_BATCH_SIZE } from "../config/env.js";
 let metaTableReady = false;
 let metaTableReadyInFlight = null;
 let globalPointsTableReady = false;
 let globalPointsTableReadyInFlight = null;
+let pointsByFileTableReady = false;
+let pointsByFileTableReadyInFlight = null;
 function throwMigrationRequired(message) {
     logger.error(message);
     throw new Error(message);
@@ -57,6 +60,10 @@ async function ensureMetaTableOnce() {
         if (!hasLastAccessedAt) {
             throwMigrationRequired("Metadata table qdr__collections is missing required column last_accessed_at; please recreate the table or apply a manual schema migration before starting the service");
         }
+        const hasUserUid = columns.some((col) => col.name === "user_uid");
+        if (!hasUserUid) {
+            throwMigrationRequired("Metadata table qdr__collections is missing required column user_uid; please recreate the table or apply a manual schema migration before starting the service");
+        }
     });
     metaTableReady = true;
 }
@@ -89,8 +96,8 @@ async function ensureGlobalPointsTableOnce() {
             // Table doesn't exist, create it with all columns using the new schema and
             // auto-partitioning enabled.
             const desc = new TableDescription()
-                .withColumns(new Column("uid", Types.UTF8), new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("embedding_quantized", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
-                .withPrimaryKeys("uid", "point_id");
+                .withColumns(new Column("collection", Types.UTF8), new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT), new Column("payload_sign", Types.UTF8), new Column("path_prefix", Types.optional(Types.UTF8)))
+                .withPrimaryKeys("collection", "point_id");
             desc.withPartitioningSettings({
                 partitioningByLoad: Ydb.FeatureFlag.Status.ENABLED,
                 partitioningBySize: Ydb.FeatureFlag.Status.ENABLED,
@@ -111,11 +118,19 @@ async function ensureGlobalPointsTableOnce() {
             // schema validation.
             tableDescription = await s.describeTable(GLOBAL_POINTS_TABLE);
         }
-        // Table exists, require the new embedding_quantized column.
+        // Table exists: validate required columns for the current schema.
         const columns = tableDescription.columns ?? [];
-        const hasEmbeddingQuantized = columns.some((col) => col.name === "embedding_quantized");
-        if (!hasEmbeddingQuantized) {
-            throwMigrationRequired(`Global points table ${GLOBAL_POINTS_TABLE} is missing required column embedding_quantized; please recreate the table or apply a manual schema migration before starting the service`);
+        const hasCollection = columns.some((col) => col.name === "collection");
+        const hasPayloadSign = columns.some((col) => col.name === "payload_sign");
+        if (!hasCollection) {
+            throwMigrationRequired(`Global points table ${GLOBAL_POINTS_TABLE} is missing required column collection; please recreate the table before starting the service`);
+        }
+        if (!hasPayloadSign) {
+            throwMigrationRequired(`Global points table ${GLOBAL_POINTS_TABLE} is missing required column payload_sign; please apply a manual schema migration before starting the service (example: ALTER TABLE ${GLOBAL_POINTS_TABLE} ADD COLUMN payload_sign Utf8)`);
+        }
+        const hasPathPrefix = columns.some((col) => col.name === "path_prefix");
+        if (!hasPathPrefix) {
+            throwMigrationRequired(`Global points table ${GLOBAL_POINTS_TABLE} is missing required column path_prefix; please apply a manual schema migration before starting the service (example: ALTER TABLE ${GLOBAL_POINTS_TABLE} ADD COLUMN path_prefix Optional<Utf8>)`);
         }
     });
 }
@@ -136,3 +151,65 @@ export async function ensureGlobalPointsTable() {
         globalPointsTableReadyInFlight = null;
     }
 }
+async function ensurePointsByFileTableOnce() {
+    await withSession(async (s) => {
+        let tableDescription = null;
+        try {
+            tableDescription = await s.describeTable(POINTS_BY_FILE_LOOKUP_TABLE);
+        }
+        catch (err) {
+            if (!isTableNotFoundError(err)) {
+                throw err;
+            }
+            const desc = new TableDescription()
+                .withColumns(new Column("collection", Types.UTF8), new Column("file_path", Types.UTF8), new Column("point_id", Types.UTF8))
+                .withPrimaryKeys("collection", "file_path", "point_id");
+            desc.withPartitioningSettings({
+                partitioningByLoad: Ydb.FeatureFlag.Status.ENABLED,
+                partitioningBySize: Ydb.FeatureFlag.Status.ENABLED,
+                partitionSizeMb: 100,
+            });
+            try {
+                await s.createTable(POINTS_BY_FILE_LOOKUP_TABLE, desc);
+                logger.info(`created points-by-file lookup table ${POINTS_BY_FILE_LOOKUP_TABLE}`);
+                return;
+            }
+            catch (createErr) {
+                if (!isAlreadyExistsError(createErr)) {
+                    throw createErr;
+                }
+            }
+            tableDescription = await s.describeTable(POINTS_BY_FILE_LOOKUP_TABLE);
+        }
+        const columns = tableDescription.columns ?? [];
+        const hasCollection = columns.some((col) => col.name === "collection");
+        const hasFilePath = columns.some((col) => col.name === "file_path");
+        const hasPointId = columns.some((col) => col.name === "point_id");
+        if (!hasCollection) {
+            throwMigrationRequired(`Points-by-file lookup table ${POINTS_BY_FILE_LOOKUP_TABLE} is missing required column collection; please recreate the table before starting the service`);
+        }
+        if (!hasFilePath) {
+            throwMigrationRequired(`Points-by-file lookup table ${POINTS_BY_FILE_LOOKUP_TABLE} is missing required column file_path; please recreate the table before starting the service`);
+        }
+        if (!hasPointId) {
+            throwMigrationRequired(`Points-by-file lookup table ${POINTS_BY_FILE_LOOKUP_TABLE} is missing required column point_id; please recreate the table before starting the service`);
+        }
+    });
+    pointsByFileTableReady = true;
+}
+export async function ensurePointsByFileTable() {
+    if (pointsByFileTableReady) {
+        return;
+    }
+    if (pointsByFileTableReadyInFlight) {
+        await pointsByFileTableReadyInFlight;
+        return;
+    }
+    pointsByFileTableReadyInFlight = ensurePointsByFileTableOnce();
+    try {
+        await pointsByFileTableReadyInFlight;
+    }
+    finally {
+        pointsByFileTableReadyInFlight = null;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ydb-qdrant",
-  "version": "8.1.1",
+  "version": "9.0.3",
   "main": "dist/package/api.js",
   "types": "dist/package/api.d.ts",
   "exports": {
@@ -16,13 +16,16 @@
   "scripts": {
     "test": "vitest run --exclude \"test/integration/**\"",
     "test:coverage": "vitest run --coverage --exclude \"test/integration/**\"",
-    "test:integration": "vitest run test/integration/YdbRealIntegration.test.ts && YDB_QDRANT_SEARCH_MODE=approximate vitest run test/integration/YdbRealIntegration.one-table.test.ts && YDB_QDRANT_SEARCH_MODE=exact vitest run test/integration/YdbRealIntegration.one-table.test.ts",
-    "test:recall": "YDB_QDRANT_SEARCH_MODE=approximate vitest run test/integration/YdbRealIntegration.one-table.test.ts && YDB_QDRANT_SEARCH_MODE=exact vitest run test/integration/YdbRealIntegration.one-table.test.ts",
+    "test:integration": "vitest run test/integration/YdbRealIntegration.test.ts && vitest run test/integration/YdbRealIntegration.one-table.test.ts",
+    "test:e2e": "vitest run --config=vitest.config.e2e.mts",
+    "test:recall": "vitest run test/integration/YdbRealIntegration.one-table.test.ts",
     "load:soak": "k6 run loadtest/soak-test.js",
     "load:stress": "k6 run loadtest/stress-test.js",
+    "load:stress:pathsegments": "k6 run loadtest/pathsegments-stress.js",
     "build": "tsc -p tsconfig.json",
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "dev": "tsx watch src/index.ts",
+    "start:e2e:app": "tsx src/index.ts",
     "start": "node --experimental-specifier-resolution=node --enable-source-maps dist/index.js",
     "smoke": "npm run build && node --experimental-specifier-resolution=node --enable-source-maps dist/SmokeTest.js",
     "lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\"",
@@ -32,7 +35,7 @@
   "keywords": [
     "ydb",
     "vector-search",
-    "approximate-nearest-neighbor",
+    "nearest-neighbor",
     "qdrant-compatible",
     "embeddings",
     "nodejs",
@@ -49,7 +52,7 @@
   ],
   "author": "",
   "license": "Apache-2.0",
-  "description": "Qdrant-compatible Node.js/TypeScript API that stores/searches embeddings in YDB using a global one-table layout with exact and approximate KNN search over serialized vectors.",
+  "description": "Qdrant-compatible Node.js/TypeScript API that stores/searches embeddings in YDB using a global one-table layout with exact KNN search over serialized vectors.",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/astandrik/ydb-qdrant.git"
@@ -69,8 +72,10 @@
     "@yandex-cloud/nodejs-sdk": "^2.9.0",
     "dotenv": "^17.2.3",
     "express": "^5.1.0",
+    "fast-json-stable-stringify": "^2.1.0",
     "nice-grpc": "^2.1.13",
     "pino": "^10.1.0",
+    "piscina": "^5.1.4",
     "ydb-sdk": "^5.11.1",
     "zod": "^4.1.12"
   },