ydb-qdrant 2.2.1 → 2.3.0

package/README.md

@@ -167,6 +167,53 @@ export async function searchDocuments(collection: string, queryEmbedding: number
 
 This pattern avoids running a separate HTTP service: vector search is executed directly against YDB via the shared `createYdbQdrantClient` instance, while the rest of your code works with plain TypeScript functions.
 
+## Recommended Vector Dimensions
+
+When creating a collection, you must specify the vector `size` matching your embedding model. Below are popular models with their dimensions and typical use cases:
+
+### Commercial API Models
+
+| Provider | Model | Dimensions | Use Cases |
+|----------|-------|------------|-----------|
+| **OpenAI** | `text-embedding-3-small` | 1536 (default, can reduce to 256-1536) | RAG, semantic search, general-purpose embeddings |
+| **OpenAI** | `text-embedding-3-large` | 3072 (default, can reduce to 256, 512, 1024, 1536, 3072) | High-accuracy RAG, multilingual tasks |
+| **OpenAI** | `text-embedding-ada-002` | 1536 | Legacy model, widely adopted |
+| **OpenAI** (Legacy) | `text-search-curie-doc-001` | 4096 | Legacy GPT-3 model, deprecated |
+| **OpenAI** (Legacy) | `text-search-davinci-doc-001` | 12288 | Legacy GPT-3 model, deprecated |
+| **Cohere** | `embed-v4.0` | 256, 512, 1024, 1536 (default) | Multimodal (text + image), RAG, enterprise search |
+| **Cohere** | `embed-english-v3.0` | 1024 | English text, semantic search, classification |
+| **Cohere** | `embed-multilingual-v3.0` | 1024 | 100+ languages, long-document retrieval, clustering |
+| **Google** | `gemini-embedding-001` | 3072 (configurable) | Multilingual, general-purpose, RAG |
+| **Google** | `text-embedding-004` | 768 | General-purpose text embeddings |
+| **Google** | `text-embedding-005` | 768 | Improved version of text-embedding-004 |
+| **Google** | `text-multilingual-embedding-002` | 768 | Multilingual text embeddings |
+
+### Open-Source Models (HuggingFace)
+
+| Model | Dimensions | Use Cases |
+|-------|------------|-----------|
+| `sentence-transformers/all-MiniLM-L6-v2` | 384 | Fast semantic search, low-resource environments |
+| `BAAI/bge-base-en-v1.5` | 768 | RAG, retrieval, English text |
+| `BAAI/bge-large-en-v1.5` | 1024 | High-accuracy RAG, English text |
+| `BAAI/bge-m3` | 1024 | Multilingual, dense/sparse/multi-vector |
+| `intfloat/e5-base-v2` | 768 | General retrieval, English text |
+| `intfloat/e5-large-v2` | 1024 | High-accuracy retrieval, English text |
+| `intfloat/e5-mistral-7b-instruct` | 4096 | High-dimensional embeddings, advanced RAG |
+| `nomic-ai/nomic-embed-text-v1` | 768 | General-purpose, open weights |
+
+### Choosing Dimensions
+
+- **Higher dimensions (1024-4096)**: Better semantic fidelity, higher storage/compute costs
+- **Lower dimensions (384-768)**: Faster queries, lower costs, suitable for many use cases
+- **Variable dimensions**: Some models (OpenAI v3, Cohere v4) allow dimension reduction with minimal accuracy loss
+- **Legacy models**: Older OpenAI GPT-3 models (Curie: 4096, Davinci: 12288) are deprecated but may still be in use
+
+**References:**
+- [OpenAI Embeddings Guide](https://platform.openai.com/docs/guides/embeddings)
+- [Cohere Embed Models](https://docs.cohere.com/docs/cohere-embed)
+- [Google Gemini Embeddings](https://ai.google.dev/gemini-api/docs/embeddings)
+- [HuggingFace Sentence Transformers](https://huggingface.co/sentence-transformers)
+
 ## Quick Start
 
 ### Use with IDE agents (Roo Code, Cline)

package/dist/config/env.d.ts

@@ -3,4 +3,4 @@ export declare const YDB_ENDPOINT: string;
 export declare const YDB_DATABASE: string;
 export declare const PORT: number;
 export declare const LOG_LEVEL: string;
-export declare const APPROX_PRESELECT: number;
+export declare const VECTOR_INDEX_BUILD_ENABLED: boolean;

package/dist/config/env.js

@@ -3,6 +3,18 @@ export const YDB_ENDPOINT = process.env.YDB_ENDPOINT ?? "";
 export const YDB_DATABASE = process.env.YDB_DATABASE ?? "";
 export const PORT = process.env.PORT ? Number(process.env.PORT) : 8080;
 export const LOG_LEVEL = process.env.LOG_LEVEL ?? "info";
-export const APPROX_PRESELECT = process.env.APPROX_PRESELECT
-    ? Math.max(1, Math.min(5000, Number(process.env.APPROX_PRESELECT)))
-    : 1000;
+function parseBooleanEnv(value, defaultValue) {
+    if (value === undefined) {
+        return defaultValue;
+    }
+    const normalized = value.trim().toLowerCase();
+    if (normalized === "" ||
+        normalized === "0" ||
+        normalized === "false" ||
+        normalized === "no" ||
+        normalized === "off") {
+        return false;
+    }
+    return true;
+}
+export const VECTOR_INDEX_BUILD_ENABLED = parseBooleanEnv(process.env.VECTOR_INDEX_BUILD_ENABLED, false);

package/dist/repositories/pointsRepo.js

@@ -2,6 +2,7 @@ import { TypedValues, withSession } from "../ydb/client.js";
 import { buildJsonOrEmpty, buildVectorParam } from "../ydb/helpers.js";
 import { logger } from "../logging/logger.js";
 import { notifyUpsert } from "../indexing/IndexScheduler.js";
+import { VECTOR_INDEX_BUILD_ENABLED } from "../config/env.js";
 export async function upsertPoints(tableName, points, vectorType, dimension) {
     let upserted = 0;
     await withSession(async (s) => {
@@ -76,26 +77,34 @@ export async function searchPoints(tableName, queryVector, top, withPayload, dis
     LIMIT $k2;
   `;
     let rs;
-    try {
-        // Try with vector index first
-        rs = await withSession(async (s) => {
-            return await s.executeQuery(buildQuery(true), params);
-        });
-        logger.info({ tableName }, "vector index found; using index for search");
-    }
-    catch (e) {
-        const msg = e instanceof Error ? e.message : String(e);
-        // Fallback to table scan if index not found or not ready
-        if (/not found|does not exist|no such index|no global index|is not ready to use/i.test(msg)) {
-            logger.info({ tableName }, "vector index not available (missing or building); falling back to table scan");
+    if (VECTOR_INDEX_BUILD_ENABLED) {
+        try {
+            // Try with vector index first
             rs = await withSession(async (s) => {
-                return await s.executeQuery(buildQuery(false), params);
+                return await s.executeQuery(buildQuery(true), params);
             });
+            logger.info({ tableName }, "vector index found; using index for search");
         }
-        else {
-            throw e;
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            const indexUnavailable = /not found|does not exist|no such index|no global index|is not ready to use/i.test(msg);
+            if (indexUnavailable) {
+                logger.info({ tableName }, "vector index not available (missing or building); falling back to table scan");
+                rs = await withSession(async (s) => {
+                    return await s.executeQuery(buildQuery(false), params);
+                });
+            }
+            else {
+                throw e;
+            }
         }
     }
+    else {
+        // Vector index usage disabled: always use table scan
+        rs = await withSession(async (s) => {
+            return await s.executeQuery(buildQuery(false), params);
+        });
+    }
     const rowset = rs.resultSets?.[0];
     const rows = (rowset?.rows ?? []);
     return rows.map((row) => {

package/dist/services/QdrantService.js

@@ -5,6 +5,7 @@ import { createCollection as repoCreateCollection, deleteCollection as repoDelet
 import { deletePoints as repoDeletePoints, searchPoints as repoSearchPoints, upsertPoints as repoUpsertPoints, } from "../repositories/pointsRepo.js";
 import { requestIndexBuild } from "../indexing/IndexScheduler.js";
 import { logger } from "../logging/logger.js";
+import { VECTOR_INDEX_BUILD_ENABLED } from "../config/env.js";
 export class QdrantServiceError extends Error {
     statusCode;
     payload;
@@ -87,6 +88,7 @@ export async function deleteCollection(ctx) {
     await repoDeleteCollection(normalized.metaKey);
     return { acknowledged: true };
 }
+let loggedIndexBuildDisabled = false;
 function isNumberArray(value) {
     return Array.isArray(value) && value.every((x) => typeof x === "number");
 }
@@ -213,7 +215,13 @@ export async function upsertPoints(ctx, body) {
         });
     }
     const upserted = await repoUpsertPoints(meta.table, parsed.data.points, meta.vectorType, meta.dimension);
-    requestIndexBuild(meta.table, meta.dimension, meta.distance, meta.vectorType);
+    if (VECTOR_INDEX_BUILD_ENABLED) {
+        requestIndexBuild(meta.table, meta.dimension, meta.distance, meta.vectorType);
+    }
+    else if (!loggedIndexBuildDisabled) {
+        logger.info({ table: meta.table }, "vector index building disabled by env; skipping automatic emb_idx rebuilds");
+        loggedIndexBuildDisabled = true;
+    }
     return { upserted };
 }
 async function executeSearch(ctx, normalizedSearch, source) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ydb-qdrant",
-  "version": "2.2.1",
+  "version": "2.3.0",
   "main": "dist/package/Api.js",
   "types": "dist/package/Api.d.ts",
   "exports": {
@@ -16,7 +16,7 @@
   "scripts": {
     "test": "vitest run --exclude \"test/integration/**\"",
     "test:coverage": "vitest run --coverage --exclude \"test/integration/**\"",
-    "test:integration": "vitest run test/integration/YdbRealIntegration.test.ts",
+    "test:integration": "VECTOR_INDEX_BUILD_ENABLED=false vitest run test/integration/YdbRealIntegration.index-disabled.test.ts && VECTOR_INDEX_BUILD_ENABLED=true vitest run test/integration/YdbRealIntegration.test.ts",
     "build": "tsc -p tsconfig.json",
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "dev": "tsx watch src/index.ts",
@@ -54,6 +54,7 @@
   "dependencies": {
     "@bufbuild/protobuf": "^2.10.0",
     "@grpc/grpc-js": "^1.14.0",
+    "@yandex-cloud/nodejs-sdk": "^2.9.0",
     "@ydbjs/api": "^6.0.4",
     "@ydbjs/core": "^6.0.4",
     "@ydbjs/query": "^6.0.4",
@@ -79,4 +80,4 @@
     "typescript-eslint": "^8.47.0",
     "vitest": "^4.0.12"
   }
-}
+}