ydb-qdrant 4.8.1 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -9
- package/dist/config/env.d.ts +1 -7
- package/dist/config/env.js +7 -27
- package/dist/index.js +10 -5
- package/dist/repositories/collectionsRepo.d.ts +2 -3
- package/dist/repositories/collectionsRepo.js +30 -62
- package/dist/repositories/pointsRepo.d.ts +3 -3
- package/dist/repositories/pointsRepo.js +3 -13
- package/dist/repositories/pointsRepo.one-table.js +0 -2
- package/dist/services/CollectionService.d.ts +0 -7
- package/dist/services/CollectionService.js +7 -21
- package/dist/services/CollectionService.one-table.d.ts +1 -1
- package/dist/services/PointsService.js +8 -17
- package/dist/utils/distance.d.ts +0 -1
- package/dist/utils/distance.js +0 -14
- package/dist/ydb/client.d.ts +7 -0
- package/dist/ydb/client.js +49 -2
- package/package.json +4 -4
- package/dist/indexing/IndexScheduler.d.ts +0 -5
- package/dist/indexing/IndexScheduler.js +0 -21
- package/dist/indexing/IndexScheduler.multi-table.d.ts +0 -12
- package/dist/indexing/IndexScheduler.multi-table.js +0 -54
- package/dist/indexing/IndexScheduler.one-table.d.ts +0 -1
- package/dist/indexing/IndexScheduler.one-table.js +0 -4
- package/dist/repositories/collectionsRepo.multi-table.d.ts +0 -3
- package/dist/repositories/collectionsRepo.multi-table.js +0 -24
- package/dist/repositories/pointsRepo.multi-table.d.ts +0 -12
- package/dist/repositories/pointsRepo.multi-table.js +0 -147
- package/dist/services/CollectionService.multi-table.d.ts +0 -5
- package/dist/services/CollectionService.multi-table.js +0 -7
package/README.md
CHANGED
|
@@ -7,9 +7,7 @@
|
|
|
7
7
|
[](https://github.com/astandrik/ydb-qdrant/actions/workflows/ci-load-stress.yml)
|
|
8
8
|
[](https://coveralls.io/github/astandrik/ydb-qdrant?branch=main)
|
|
9
9
|
|
|
10
|
-
[](https://github.com/astandrik/ydb-qdrant/actions/workflows/ci-recall.yml)
|
|
11
10
|
[](https://github.com/astandrik/ydb-qdrant/actions/workflows/ci-recall.yml)
|
|
12
|
-
[](https://github.com/astandrik/ydb-qdrant/actions/workflows/ci-recall.yml)
|
|
13
11
|
[](https://github.com/astandrik/ydb-qdrant/actions/workflows/ci-recall.yml)
|
|
14
12
|
|
|
15
13
|
[](https://www.npmjs.com/package/ydb-qdrant)
|
|
@@ -18,7 +16,7 @@
|
|
|
18
16
|
|
|
19
17
|
# YDB Qdrant-compatible Service
|
|
20
18
|
|
|
21
|
-
Qdrant-compatible Node.js/TypeScript **service and npm library** that stores and searches vectors in YDB using
|
|
19
|
+
Qdrant-compatible Node.js/TypeScript **service and npm library** that stores and searches vectors in YDB using a global one-table layout (`qdrant_all_points`) with exact KNN search (single-phase over `embedding`) by default and an optional approximate mode (two‑phase bit-quantized over `embedding_quantized` + `embedding`). Topics: ydb, vector-search, qdrant-compatible, nodejs, typescript, express, yql, ann, semantic-search, rag.
|
|
22
20
|
|
|
23
21
|
Modes:
|
|
24
22
|
- **HTTP server**: Qdrant-compatible REST API (`/collections`, `/points/*`) on top of YDB.
|
|
@@ -35,7 +33,7 @@ Architecture diagrams: [docs page](http://ydb-qdrant.tech/docs/)
|
|
|
35
33
|
|
|
36
34
|
- **Vector dimensions and embedding models**: [docs/vector-dimensions.md](docs/vector-dimensions.md)
|
|
37
35
|
- **Deployment and Docker options**: [docs/deployment-and-docker.md](docs/deployment-and-docker.md)
|
|
38
|
-
- **Architecture, storage layout, and
|
|
36
|
+
- **Architecture, storage layout, and search modes**: [docs/architecture-and-storage.md](docs/architecture-and-storage.md)
|
|
39
37
|
- **Evaluation, CI, and release process**: [docs/evaluation-and-ci.md](docs/evaluation-and-ci.md)
|
|
40
38
|
|
|
41
39
|
## Requirements
|
|
@@ -90,10 +88,8 @@ Optional env:
|
|
|
90
88
|
# Server
|
|
91
89
|
export PORT=8080
|
|
92
90
|
export LOG_LEVEL=info
|
|
93
|
-
#
|
|
94
|
-
export
|
|
95
|
-
# One-table search tuning (one_table mode only)
|
|
96
|
-
export YDB_QDRANT_SEARCH_MODE=approximate # or exact
|
|
91
|
+
# One-table search tuning (default is 'exact' when unset)
|
|
92
|
+
export YDB_QDRANT_SEARCH_MODE=approximate # approximate or exact (default: exact)
|
|
97
93
|
export YDB_QDRANT_OVERFETCH_MULTIPLIER=10 # candidate multiplier in approximate mode
|
|
98
94
|
```
|
|
99
95
|
|
|
@@ -334,7 +330,7 @@ curl -X POST http://localhost:8080/collections/mycol/points/delete \
|
|
|
334
330
|
|
|
335
331
|
## Architecture and Storage
|
|
336
332
|
|
|
337
|
-
For details on the YDB storage layout (
|
|
333
|
+
For details on the YDB one-table storage layout, vector serialization (full-precision and bit‑quantized), approximate vs exact search modes, request normalization, and Qdrant compatibility semantics, see [docs/architecture-and-storage.md](docs/architecture-and-storage.md).
|
|
338
334
|
|
|
339
335
|
## Evaluation, CI, and Release
|
|
340
336
|
|
package/dist/config/env.d.ts
CHANGED
|
@@ -3,18 +3,12 @@ export declare const YDB_ENDPOINT: string;
|
|
|
3
3
|
export declare const YDB_DATABASE: string;
|
|
4
4
|
export declare const PORT: number;
|
|
5
5
|
export declare const LOG_LEVEL: string;
|
|
6
|
-
export declare enum CollectionStorageMode {
|
|
7
|
-
MultiTable = "multi_table",
|
|
8
|
-
OneTable = "one_table"
|
|
9
|
-
}
|
|
10
|
-
export declare const COLLECTION_STORAGE_MODE: CollectionStorageMode;
|
|
11
6
|
export declare const GLOBAL_POINTS_AUTOMIGRATE_ENABLED: boolean;
|
|
12
|
-
export declare const VECTOR_INDEX_BUILD_ENABLED: boolean;
|
|
13
|
-
export declare function isOneTableMode(mode: CollectionStorageMode): mode is CollectionStorageMode.OneTable;
|
|
14
7
|
export declare enum SearchMode {
|
|
15
8
|
Exact = "exact",
|
|
16
9
|
Approximate = "approximate"
|
|
17
10
|
}
|
|
11
|
+
export declare function resolveSearchMode(raw: string | undefined): SearchMode;
|
|
18
12
|
export declare const SEARCH_MODE: SearchMode;
|
|
19
13
|
export declare const OVERFETCH_MULTIPLIER: number;
|
|
20
14
|
export declare const CLIENT_SIDE_SERIALIZATION_ENABLED: boolean;
|
package/dist/config/env.js
CHANGED
|
@@ -34,32 +34,13 @@ function parseBooleanEnv(value, defaultValue) {
|
|
|
34
34
|
}
|
|
35
35
|
return true;
|
|
36
36
|
}
|
|
37
|
-
export var CollectionStorageMode;
|
|
38
|
-
(function (CollectionStorageMode) {
|
|
39
|
-
CollectionStorageMode["MultiTable"] = "multi_table";
|
|
40
|
-
CollectionStorageMode["OneTable"] = "one_table";
|
|
41
|
-
})(CollectionStorageMode || (CollectionStorageMode = {}));
|
|
42
|
-
function resolveCollectionStorageModeEnv() {
|
|
43
|
-
const explicit = process.env.YDB_QDRANT_COLLECTION_STORAGE_MODE ??
|
|
44
|
-
process.env.YDB_QDRANT_TABLE_LAYOUT;
|
|
45
|
-
if (explicit?.trim().toLowerCase() === CollectionStorageMode.OneTable) {
|
|
46
|
-
return CollectionStorageMode.OneTable;
|
|
47
|
-
}
|
|
48
|
-
return CollectionStorageMode.MultiTable;
|
|
49
|
-
}
|
|
50
|
-
export const COLLECTION_STORAGE_MODE = resolveCollectionStorageModeEnv();
|
|
51
37
|
export const GLOBAL_POINTS_AUTOMIGRATE_ENABLED = parseBooleanEnv(process.env.YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE, false);
|
|
52
|
-
export const VECTOR_INDEX_BUILD_ENABLED = parseBooleanEnv(process.env.VECTOR_INDEX_BUILD_ENABLED, COLLECTION_STORAGE_MODE === CollectionStorageMode.MultiTable);
|
|
53
|
-
export function isOneTableMode(mode) {
|
|
54
|
-
return mode === CollectionStorageMode.OneTable;
|
|
55
|
-
}
|
|
56
38
|
export var SearchMode;
|
|
57
39
|
(function (SearchMode) {
|
|
58
40
|
SearchMode["Exact"] = "exact";
|
|
59
41
|
SearchMode["Approximate"] = "approximate";
|
|
60
42
|
})(SearchMode || (SearchMode = {}));
|
|
61
|
-
function
|
|
62
|
-
const raw = process.env.YDB_QDRANT_SEARCH_MODE;
|
|
43
|
+
export function resolveSearchMode(raw) {
|
|
63
44
|
const normalized = raw?.trim().toLowerCase();
|
|
64
45
|
if (normalized === SearchMode.Exact) {
|
|
65
46
|
return SearchMode.Exact;
|
|
@@ -67,14 +48,13 @@ function resolveSearchModeEnv(mode) {
|
|
|
67
48
|
if (normalized === SearchMode.Approximate) {
|
|
68
49
|
return SearchMode.Approximate;
|
|
69
50
|
}
|
|
70
|
-
// Default:
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
return SearchMode.Approximate;
|
|
51
|
+
// Default: exact search (single-phase over full-precision embedding) for the one-table layout.
|
|
52
|
+
return SearchMode.Exact;
|
|
53
|
+
}
|
|
54
|
+
function resolveSearchModeEnv() {
|
|
55
|
+
return resolveSearchMode(process.env.YDB_QDRANT_SEARCH_MODE);
|
|
76
56
|
}
|
|
77
|
-
export const SEARCH_MODE = resolveSearchModeEnv(
|
|
57
|
+
export const SEARCH_MODE = resolveSearchModeEnv();
|
|
78
58
|
export const OVERFETCH_MULTIPLIER = parseIntegerEnv(process.env.YDB_QDRANT_OVERFETCH_MULTIPLIER, 10, { min: 1 });
|
|
79
59
|
export const CLIENT_SIDE_SERIALIZATION_ENABLED = parseBooleanEnv(process.env.YDB_QDRANT_CLIENT_SIDE_SERIALIZATION_ENABLED, false);
|
|
80
60
|
export const UPSERT_BATCH_SIZE = parseIntegerEnv(process.env.YDB_QDRANT_UPSERT_BATCH_SIZE, 100, { min: 1 });
|
package/dist/index.js
CHANGED
|
@@ -1,18 +1,23 @@
|
|
|
1
1
|
import "dotenv/config";
|
|
2
2
|
import { buildServer } from "./server.js";
|
|
3
|
-
import { PORT
|
|
3
|
+
import { PORT } from "./config/env.js";
|
|
4
4
|
import { logger } from "./logging/logger.js";
|
|
5
|
-
import { readyOrThrow } from "./ydb/client.js";
|
|
5
|
+
import { readyOrThrow, isCompilationTimeoutError } from "./ydb/client.js";
|
|
6
6
|
import { ensureMetaTable, ensureGlobalPointsTable } from "./ydb/schema.js";
|
|
7
|
+
import { verifyCollectionsQueryCompilationForStartup } from "./repositories/collectionsRepo.js";
|
|
7
8
|
async function start() {
|
|
8
9
|
try {
|
|
9
10
|
await readyOrThrow();
|
|
10
11
|
await ensureMetaTable();
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
await ensureGlobalPointsTable();
|
|
13
|
+
await verifyCollectionsQueryCompilationForStartup();
|
|
14
|
+
logger.info("YDB compilation startup probe for qdr__collections completed successfully");
|
|
14
15
|
}
|
|
15
16
|
catch (err) {
|
|
17
|
+
if (isCompilationTimeoutError(err)) {
|
|
18
|
+
logger.error({ err }, "Fatal YDB compilation timeout during startup probe; exiting so supervisor can restart the process");
|
|
19
|
+
process.exit(1);
|
|
20
|
+
}
|
|
16
21
|
logger.error({ err }, "YDB not ready; startup continues, requests may fail until configured.");
|
|
17
22
|
}
|
|
18
23
|
const app = buildServer();
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import type { DistanceKind, VectorType } from "../types";
|
|
2
|
-
|
|
3
|
-
export declare function createCollection(metaKey: string, dim: number, distance: DistanceKind, vectorType: VectorType, tableName: string, layout?: CollectionStorageMode): Promise<void>;
|
|
2
|
+
export declare function createCollection(metaKey: string, dim: number, distance: DistanceKind, vectorType: VectorType): Promise<void>;
|
|
4
3
|
export declare function getCollectionMeta(metaKey: string): Promise<{
|
|
5
4
|
table: string;
|
|
6
5
|
dimension: number;
|
|
7
6
|
distance: DistanceKind;
|
|
8
7
|
vectorType: VectorType;
|
|
9
8
|
} | null>;
|
|
9
|
+
export declare function verifyCollectionsQueryCompilationForStartup(): Promise<void>;
|
|
10
10
|
export declare function deleteCollection(metaKey: string, uid?: string): Promise<void>;
|
|
11
|
-
export declare function buildVectorIndex(tableName: string, dimension: number, distance: DistanceKind, vectorType: VectorType): Promise<void>;
|
|
@@ -1,16 +1,8 @@
|
|
|
1
|
-
import { TypedValues, withSession, createExecuteQuerySettings, } from "../ydb/client.js";
|
|
2
|
-
import { mapDistanceToIndexParam } from "../utils/distance.js";
|
|
3
|
-
import { COLLECTION_STORAGE_MODE, isOneTableMode, } from "../config/env.js";
|
|
4
|
-
import { GLOBAL_POINTS_TABLE } from "../ydb/schema.js";
|
|
1
|
+
import { TypedValues, withSession, createExecuteQuerySettings, withStartupProbeSession, createExecuteQuerySettingsWithTimeout, } from "../ydb/client.js";
|
|
5
2
|
import { uidFor } from "../utils/tenant.js";
|
|
6
|
-
import { createCollectionMultiTable, deleteCollectionMultiTable, } from "./collectionsRepo.multi-table.js";
|
|
7
3
|
import { createCollectionOneTable, deleteCollectionOneTable, } from "./collectionsRepo.one-table.js";
|
|
8
|
-
export async function createCollection(metaKey, dim, distance, vectorType
|
|
9
|
-
|
|
10
|
-
await createCollectionOneTable(metaKey, dim, distance, vectorType);
|
|
11
|
-
return;
|
|
12
|
-
}
|
|
13
|
-
await createCollectionMultiTable(metaKey, dim, distance, vectorType, tableName);
|
|
4
|
+
export async function createCollection(metaKey, dim, distance, vectorType) {
|
|
5
|
+
await createCollectionOneTable(metaKey, dim, distance, vectorType);
|
|
14
6
|
}
|
|
15
7
|
export async function getCollectionMeta(metaKey) {
|
|
16
8
|
const qry = `
|
|
@@ -35,60 +27,36 @@ export async function getCollectionMeta(metaKey) {
|
|
|
35
27
|
const vectorType = row.items?.[3]?.textValue ?? "float";
|
|
36
28
|
return { table, dimension, distance, vectorType };
|
|
37
29
|
}
|
|
30
|
+
export async function verifyCollectionsQueryCompilationForStartup() {
|
|
31
|
+
const probeKey = "__startup_probe__/__startup_probe__";
|
|
32
|
+
const qry = `
|
|
33
|
+
DECLARE $collection AS Utf8;
|
|
34
|
+
SELECT table_name, vector_dimension, distance, vector_type
|
|
35
|
+
FROM qdr__collections
|
|
36
|
+
WHERE collection = $collection;
|
|
37
|
+
`;
|
|
38
|
+
await withStartupProbeSession(async (s) => {
|
|
39
|
+
const settings = createExecuteQuerySettingsWithTimeout({
|
|
40
|
+
keepInCache: true,
|
|
41
|
+
idempotent: true,
|
|
42
|
+
timeoutMs: 3000,
|
|
43
|
+
});
|
|
44
|
+
await s.executeQuery(qry, {
|
|
45
|
+
$collection: TypedValues.utf8(probeKey),
|
|
46
|
+
}, undefined, settings);
|
|
47
|
+
});
|
|
48
|
+
}
|
|
38
49
|
export async function deleteCollection(metaKey, uid) {
|
|
39
50
|
const meta = await getCollectionMeta(metaKey);
|
|
40
51
|
if (!meta)
|
|
41
52
|
return;
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
throw new Error(`deleteCollection: cannot derive uid from malformed metaKey=${metaKey}`);
|
|
48
|
-
}
|
|
49
|
-
return uidFor(tenant, collection);
|
|
50
|
-
})();
|
|
51
|
-
await deleteCollectionOneTable(metaKey, effectiveUid);
|
|
52
|
-
return;
|
|
53
|
-
}
|
|
54
|
-
await deleteCollectionMultiTable(metaKey, meta.table);
|
|
55
|
-
}
|
|
56
|
-
export async function buildVectorIndex(tableName, dimension, distance, vectorType) {
|
|
57
|
-
const distParam = mapDistanceToIndexParam(distance);
|
|
58
|
-
// defaults for <100k vectors
|
|
59
|
-
const levels = 1;
|
|
60
|
-
const clusters = 128;
|
|
61
|
-
await withSession(async (s) => {
|
|
62
|
-
// Drop existing index if present
|
|
63
|
-
const dropDdl = `ALTER TABLE ${tableName} DROP INDEX emb_idx;`;
|
|
64
|
-
const rawSession = s;
|
|
65
|
-
try {
|
|
66
|
-
const dropReq = { sessionId: rawSession.sessionId, yqlText: dropDdl };
|
|
67
|
-
await rawSession.api.executeSchemeQuery(dropReq);
|
|
68
|
-
}
|
|
69
|
-
catch (e) {
|
|
70
|
-
const msg = e instanceof Error ? e.message : String(e);
|
|
71
|
-
// ignore if index doesn't exist
|
|
72
|
-
if (!/not found|does not exist|no such index/i.test(msg)) {
|
|
73
|
-
throw e;
|
|
74
|
-
}
|
|
53
|
+
let effectiveUid = uid;
|
|
54
|
+
if (!effectiveUid) {
|
|
55
|
+
const [tenant, collection] = metaKey.split("/", 2);
|
|
56
|
+
if (!tenant || !collection) {
|
|
57
|
+
throw new Error(`deleteCollection: cannot derive uid from malformed metaKey=${metaKey}`);
|
|
75
58
|
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
ADD INDEX emb_idx GLOBAL SYNC USING vector_kmeans_tree
|
|
80
|
-
ON (embedding)
|
|
81
|
-
WITH (
|
|
82
|
-
${distParam === "inner_product"
|
|
83
|
-
? `similarity="inner_product"`
|
|
84
|
-
: `distance="${distParam}"`},
|
|
85
|
-
vector_type="${vectorType}",
|
|
86
|
-
vector_dimension=${dimension},
|
|
87
|
-
clusters=${clusters},
|
|
88
|
-
levels=${levels}
|
|
89
|
-
);
|
|
90
|
-
`;
|
|
91
|
-
const createReq = { sessionId: rawSession.sessionId, yqlText: createDdl };
|
|
92
|
-
await rawSession.api.executeSchemeQuery(createReq);
|
|
93
|
-
});
|
|
59
|
+
effectiveUid = uidFor(tenant, collection);
|
|
60
|
+
}
|
|
61
|
+
await deleteCollectionOneTable(metaKey, effectiveUid);
|
|
94
62
|
}
|
|
@@ -3,10 +3,10 @@ export declare function upsertPoints(tableName: string, points: Array<{
|
|
|
3
3
|
id: string | number;
|
|
4
4
|
vector: number[];
|
|
5
5
|
payload?: Record<string, unknown>;
|
|
6
|
-
}>, dimension: number, uid
|
|
7
|
-
export declare function searchPoints(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number, uid
|
|
6
|
+
}>, dimension: number, uid: string): Promise<number>;
|
|
7
|
+
export declare function searchPoints(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number, uid: string): Promise<Array<{
|
|
8
8
|
id: string;
|
|
9
9
|
score: number;
|
|
10
10
|
payload?: Record<string, unknown>;
|
|
11
11
|
}>>;
|
|
12
|
-
export declare function deletePoints(tableName: string, ids: Array<string | number>, uid
|
|
12
|
+
export declare function deletePoints(tableName: string, ids: Array<string | number>, uid: string): Promise<number>;
|
|
@@ -1,22 +1,12 @@
|
|
|
1
|
-
import { upsertPointsMultiTable, searchPointsMultiTable, deletePointsMultiTable, } from "./pointsRepo.multi-table.js";
|
|
2
1
|
import { SEARCH_MODE, OVERFETCH_MULTIPLIER, } from "../config/env.js";
|
|
3
2
|
import { upsertPointsOneTable, searchPointsOneTable, deletePointsOneTable, } from "./pointsRepo.one-table.js";
|
|
4
3
|
export async function upsertPoints(tableName, points, dimension, uid) {
|
|
5
|
-
|
|
6
|
-
return await upsertPointsOneTable(tableName, points, dimension, uid);
|
|
7
|
-
}
|
|
8
|
-
return await upsertPointsMultiTable(tableName, points, dimension);
|
|
4
|
+
return await upsertPointsOneTable(tableName, points, dimension, uid);
|
|
9
5
|
}
|
|
10
6
|
export async function searchPoints(tableName, queryVector, top, withPayload, distance, dimension, uid) {
|
|
11
7
|
const mode = SEARCH_MODE;
|
|
12
|
-
|
|
13
|
-
return await searchPointsOneTable(tableName, queryVector, top, withPayload, distance, dimension, uid, mode, OVERFETCH_MULTIPLIER);
|
|
14
|
-
}
|
|
15
|
-
return await searchPointsMultiTable(tableName, queryVector, top, withPayload, distance, dimension);
|
|
8
|
+
return await searchPointsOneTable(tableName, queryVector, top, withPayload, distance, dimension, uid, mode, OVERFETCH_MULTIPLIER);
|
|
16
9
|
}
|
|
17
10
|
export async function deletePoints(tableName, ids, uid) {
|
|
18
|
-
|
|
19
|
-
return await deletePointsOneTable(tableName, ids, uid);
|
|
20
|
-
}
|
|
21
|
-
return await deletePointsMultiTable(tableName, ids);
|
|
11
|
+
return await deletePointsOneTable(tableName, ids, uid);
|
|
22
12
|
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { TypedValues, Types, withSession, createExecuteQuerySettings, } from "../ydb/client.js";
|
|
2
2
|
import { buildVectorParam, buildVectorBinaryParams } from "../ydb/helpers.js";
|
|
3
|
-
import { notifyUpsert } from "../indexing/IndexScheduler.js";
|
|
4
3
|
import { mapDistanceToKnnFn, mapDistanceToBitKnnFn, } from "../utils/distance.js";
|
|
5
4
|
import { withRetry, isTransientYdbError } from "../utils/retry.js";
|
|
6
5
|
import { UPSERT_BATCH_SIZE } from "../ydb/schema.js";
|
|
@@ -128,7 +127,6 @@ export async function upsertPointsOneTable(tableName, points, dimension, uid) {
|
|
|
128
127
|
upserted += batch.length;
|
|
129
128
|
}
|
|
130
129
|
});
|
|
131
|
-
notifyUpsert(tableName, upserted);
|
|
132
130
|
return upserted;
|
|
133
131
|
}
|
|
134
132
|
async function searchPointsOneTableExact(tableName, queryVector, top, withPayload, distance, dimension, uid) {
|
|
@@ -10,13 +10,6 @@ export interface NormalizedCollectionContext {
|
|
|
10
10
|
collection: string;
|
|
11
11
|
metaKey: string;
|
|
12
12
|
}
|
|
13
|
-
export declare function normalizeCollectionContext(input: CollectionContextInput): NormalizedCollectionContext;
|
|
14
|
-
export declare function resolvePointsTableAndUid(ctx: NormalizedCollectionContext, meta: {
|
|
15
|
-
table: string;
|
|
16
|
-
}): Promise<{
|
|
17
|
-
tableName: string;
|
|
18
|
-
uid: string | undefined;
|
|
19
|
-
}>;
|
|
20
13
|
export declare function putCollectionIndex(ctx: CollectionContextInput): Promise<{
|
|
21
14
|
acknowledged: boolean;
|
|
22
15
|
}>;
|
|
@@ -1,24 +1,11 @@
|
|
|
1
1
|
import { CreateCollectionReq } from "../types.js";
|
|
2
|
-
import { ensureMetaTable
|
|
2
|
+
import { ensureMetaTable } from "../ydb/schema.js";
|
|
3
3
|
import { createCollection as repoCreateCollection, deleteCollection as repoDeleteCollection, getCollectionMeta, } from "../repositories/collectionsRepo.js";
|
|
4
4
|
import { QdrantServiceError } from "./errors.js";
|
|
5
|
-
import { normalizeCollectionContextShared
|
|
6
|
-
import { resolvePointsTableAndUidOneTable } from "./CollectionService.one-table.js";
|
|
7
|
-
export function normalizeCollectionContext(input) {
|
|
8
|
-
return normalizeCollectionContextShared(input.tenant, input.collection, input.apiKey, input.userAgent);
|
|
9
|
-
}
|
|
10
|
-
export async function resolvePointsTableAndUid(ctx, meta) {
|
|
11
|
-
if (meta?.table === GLOBAL_POINTS_TABLE) {
|
|
12
|
-
return await resolvePointsTableAndUidOneTable(ctx);
|
|
13
|
-
}
|
|
14
|
-
return {
|
|
15
|
-
tableName: meta.table,
|
|
16
|
-
uid: undefined,
|
|
17
|
-
};
|
|
18
|
-
}
|
|
5
|
+
import { normalizeCollectionContextShared } from "./CollectionService.shared.js";
|
|
19
6
|
export async function putCollectionIndex(ctx) {
|
|
20
7
|
await ensureMetaTable();
|
|
21
|
-
const normalized =
|
|
8
|
+
const normalized = normalizeCollectionContextShared(ctx.tenant, ctx.collection, ctx.apiKey, ctx.userAgent);
|
|
22
9
|
const meta = await getCollectionMeta(normalized.metaKey);
|
|
23
10
|
if (!meta) {
|
|
24
11
|
throw new QdrantServiceError(404, {
|
|
@@ -30,7 +17,7 @@ export async function putCollectionIndex(ctx) {
|
|
|
30
17
|
}
|
|
31
18
|
export async function createCollection(ctx, body) {
|
|
32
19
|
await ensureMetaTable();
|
|
33
|
-
const normalized =
|
|
20
|
+
const normalized = normalizeCollectionContextShared(ctx.tenant, ctx.collection, ctx.apiKey, ctx.userAgent);
|
|
34
21
|
const parsed = CreateCollectionReq.safeParse(body);
|
|
35
22
|
if (!parsed.success) {
|
|
36
23
|
throw new QdrantServiceError(400, {
|
|
@@ -54,13 +41,12 @@ export async function createCollection(ctx, body) {
|
|
|
54
41
|
error: errorMessage,
|
|
55
42
|
});
|
|
56
43
|
}
|
|
57
|
-
|
|
58
|
-
await repoCreateCollection(normalized.metaKey, dim, distance, vectorType, tableName);
|
|
44
|
+
await repoCreateCollection(normalized.metaKey, dim, distance, vectorType);
|
|
59
45
|
return { name: normalized.collection, tenant: normalized.tenant };
|
|
60
46
|
}
|
|
61
47
|
export async function getCollection(ctx) {
|
|
62
48
|
await ensureMetaTable();
|
|
63
|
-
const normalized =
|
|
49
|
+
const normalized = normalizeCollectionContextShared(ctx.tenant, ctx.collection, ctx.apiKey, ctx.userAgent);
|
|
64
50
|
const meta = await getCollectionMeta(normalized.metaKey);
|
|
65
51
|
if (!meta) {
|
|
66
52
|
throw new QdrantServiceError(404, {
|
|
@@ -79,7 +65,7 @@ export async function getCollection(ctx) {
|
|
|
79
65
|
}
|
|
80
66
|
export async function deleteCollection(ctx) {
|
|
81
67
|
await ensureMetaTable();
|
|
82
|
-
const normalized =
|
|
68
|
+
const normalized = normalizeCollectionContextShared(ctx.tenant, ctx.collection, ctx.apiKey, ctx.userAgent);
|
|
83
69
|
await repoDeleteCollection(normalized.metaKey);
|
|
84
70
|
return { acknowledged: true };
|
|
85
71
|
}
|
|
@@ -2,16 +2,14 @@ import { UpsertPointsReq, SearchReq, DeletePointsReq } from "../types.js";
|
|
|
2
2
|
import { ensureMetaTable } from "../ydb/schema.js";
|
|
3
3
|
import { getCollectionMeta } from "../repositories/collectionsRepo.js";
|
|
4
4
|
import { deletePoints as repoDeletePoints, searchPoints as repoSearchPoints, upsertPoints as repoUpsertPoints, } from "../repositories/pointsRepo.js";
|
|
5
|
-
import { requestIndexBuild } from "../indexing/IndexScheduler.js";
|
|
6
5
|
import { logger } from "../logging/logger.js";
|
|
7
|
-
import { VECTOR_INDEX_BUILD_ENABLED } from "../config/env.js";
|
|
8
6
|
import { QdrantServiceError, isVectorDimensionMismatchError, } from "./errors.js";
|
|
9
|
-
import {
|
|
7
|
+
import { normalizeCollectionContextShared } from "./CollectionService.shared.js";
|
|
8
|
+
import { resolvePointsTableAndUidOneTable } from "./CollectionService.one-table.js";
|
|
10
9
|
import { normalizeSearchBodyForSearch, normalizeSearchBodyForQuery, } from "../utils/normalization.js";
|
|
11
|
-
let loggedIndexBuildDisabled = false;
|
|
12
10
|
export async function upsertPoints(ctx, body) {
|
|
13
11
|
await ensureMetaTable();
|
|
14
|
-
const normalized =
|
|
12
|
+
const normalized = normalizeCollectionContextShared(ctx.tenant, ctx.collection, ctx.apiKey, ctx.userAgent);
|
|
15
13
|
const meta = await getCollectionMeta(normalized.metaKey);
|
|
16
14
|
if (!meta) {
|
|
17
15
|
throw new QdrantServiceError(404, {
|
|
@@ -26,7 +24,7 @@ export async function upsertPoints(ctx, body) {
|
|
|
26
24
|
error: parsed.error.flatten(),
|
|
27
25
|
});
|
|
28
26
|
}
|
|
29
|
-
const { tableName, uid } = await
|
|
27
|
+
const { tableName, uid } = await resolvePointsTableAndUidOneTable(normalized);
|
|
30
28
|
let upserted;
|
|
31
29
|
try {
|
|
32
30
|
upserted = await repoUpsertPoints(tableName, parsed.data.points, meta.dimension, uid);
|
|
@@ -46,18 +44,11 @@ export async function upsertPoints(ctx, body) {
|
|
|
46
44
|
}
|
|
47
45
|
throw err;
|
|
48
46
|
}
|
|
49
|
-
if (VECTOR_INDEX_BUILD_ENABLED) {
|
|
50
|
-
requestIndexBuild(tableName, meta.dimension, meta.distance, meta.vectorType);
|
|
51
|
-
}
|
|
52
|
-
else if (!loggedIndexBuildDisabled) {
|
|
53
|
-
logger.info({ table: tableName }, "vector index building disabled by env; skipping automatic emb_idx rebuilds");
|
|
54
|
-
loggedIndexBuildDisabled = true;
|
|
55
|
-
}
|
|
56
47
|
return { upserted };
|
|
57
48
|
}
|
|
58
49
|
async function executeSearch(ctx, normalizedSearch, source) {
|
|
59
50
|
await ensureMetaTable();
|
|
60
|
-
const normalized =
|
|
51
|
+
const normalized = normalizeCollectionContextShared(ctx.tenant, ctx.collection, ctx.apiKey, ctx.userAgent);
|
|
61
52
|
logger.info({ tenant: normalized.tenant, collection: normalized.collection }, `${source}: resolve collection meta`);
|
|
62
53
|
const meta = await getCollectionMeta(normalized.metaKey);
|
|
63
54
|
if (!meta) {
|
|
@@ -87,7 +78,7 @@ async function executeSearch(ctx, normalizedSearch, source) {
|
|
|
87
78
|
error: parsed.error.flatten(),
|
|
88
79
|
});
|
|
89
80
|
}
|
|
90
|
-
const { tableName, uid } = await
|
|
81
|
+
const { tableName, uid } = await resolvePointsTableAndUidOneTable(normalized);
|
|
91
82
|
logger.info({
|
|
92
83
|
tenant: normalized.tenant,
|
|
93
84
|
collection: normalized.collection,
|
|
@@ -154,7 +145,7 @@ export async function queryPoints(ctx, body) {
|
|
|
154
145
|
}
|
|
155
146
|
export async function deletePoints(ctx, body) {
|
|
156
147
|
await ensureMetaTable();
|
|
157
|
-
const normalized =
|
|
148
|
+
const normalized = normalizeCollectionContextShared(ctx.tenant, ctx.collection, ctx.apiKey, ctx.userAgent);
|
|
158
149
|
const meta = await getCollectionMeta(normalized.metaKey);
|
|
159
150
|
if (!meta) {
|
|
160
151
|
throw new QdrantServiceError(404, {
|
|
@@ -169,7 +160,7 @@ export async function deletePoints(ctx, body) {
|
|
|
169
160
|
error: parsed.error.flatten(),
|
|
170
161
|
});
|
|
171
162
|
}
|
|
172
|
-
const { tableName, uid } = await
|
|
163
|
+
const { tableName, uid } = await resolvePointsTableAndUidOneTable(normalized);
|
|
173
164
|
const deleted = await repoDeletePoints(tableName, parsed.data.points, uid);
|
|
174
165
|
return { deleted };
|
|
175
166
|
}
|
package/dist/utils/distance.d.ts
CHANGED
|
@@ -3,7 +3,6 @@ export declare function mapDistanceToKnnFn(distance: DistanceKind): {
|
|
|
3
3
|
fn: string;
|
|
4
4
|
order: "ASC" | "DESC";
|
|
5
5
|
};
|
|
6
|
-
export declare function mapDistanceToIndexParam(distance: DistanceKind): string;
|
|
7
6
|
/**
|
|
8
7
|
* Maps a user-specified distance metric to a YDB Knn function
|
|
9
8
|
* suitable for bit-quantized vectors (Phase 1 approximate candidate selection).
|
package/dist/utils/distance.js
CHANGED
|
@@ -12,20 +12,6 @@ export function mapDistanceToKnnFn(distance) {
|
|
|
12
12
|
return { fn: "Knn::CosineDistance", order: "ASC" };
|
|
13
13
|
}
|
|
14
14
|
}
|
|
15
|
-
export function mapDistanceToIndexParam(distance) {
|
|
16
|
-
switch (distance) {
|
|
17
|
-
case "Cosine":
|
|
18
|
-
return "cosine";
|
|
19
|
-
case "Dot":
|
|
20
|
-
return "inner_product";
|
|
21
|
-
case "Euclid":
|
|
22
|
-
return "euclidean";
|
|
23
|
-
case "Manhattan":
|
|
24
|
-
return "manhattan";
|
|
25
|
-
default:
|
|
26
|
-
return "cosine";
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
15
|
/**
|
|
30
16
|
* Maps a user-specified distance metric to a YDB Knn function
|
|
31
17
|
* suitable for bit-quantized vectors (Phase 1 approximate candidate selection).
|
package/dist/ydb/client.d.ts
CHANGED
|
@@ -5,18 +5,25 @@ export declare function createExecuteQuerySettings(options?: {
|
|
|
5
5
|
keepInCache?: boolean;
|
|
6
6
|
idempotent?: boolean;
|
|
7
7
|
}): YdbExecuteQuerySettings;
|
|
8
|
+
export declare function createExecuteQuerySettingsWithTimeout(options: {
|
|
9
|
+
keepInCache?: boolean;
|
|
10
|
+
idempotent?: boolean;
|
|
11
|
+
timeoutMs: number;
|
|
12
|
+
}): YdbExecuteQuerySettings;
|
|
8
13
|
type DriverConfig = {
|
|
9
14
|
endpoint?: string;
|
|
10
15
|
database?: string;
|
|
11
16
|
connectionString?: string;
|
|
12
17
|
authService?: IAuthService;
|
|
13
18
|
};
|
|
19
|
+
export declare function isCompilationTimeoutError(error: unknown): boolean;
|
|
14
20
|
export declare function __setDriverForTests(fake: unknown): void;
|
|
15
21
|
export declare function __setDriverFactoryForTests(factory: ((config: unknown) => unknown) | undefined): void;
|
|
16
22
|
export declare function __resetRefreshStateForTests(): void;
|
|
17
23
|
export declare function configureDriver(config: DriverConfig): void;
|
|
18
24
|
export declare function readyOrThrow(): Promise<void>;
|
|
19
25
|
export declare function withSession<T>(fn: (s: Session) => Promise<T>): Promise<T>;
|
|
26
|
+
export declare function withStartupProbeSession<T>(fn: (s: Session) => Promise<T>): Promise<T>;
|
|
20
27
|
export declare function isYdbAvailable(timeoutMs?: number): Promise<boolean>;
|
|
21
28
|
/**
|
|
22
29
|
* Destroys the current driver and its session pool.
|
package/dist/ydb/client.js
CHANGED
|
@@ -2,7 +2,7 @@ import { createRequire } from "module";
|
|
|
2
2
|
import { YDB_DATABASE, YDB_ENDPOINT, SESSION_POOL_MIN_SIZE, SESSION_POOL_MAX_SIZE, SESSION_KEEPALIVE_PERIOD_MS, } from "../config/env.js";
|
|
3
3
|
import { logger } from "../logging/logger.js";
|
|
4
4
|
const require = createRequire(import.meta.url);
|
|
5
|
-
const { Driver, getCredentialsFromEnv, Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, } = require("ydb-sdk");
|
|
5
|
+
const { Driver, getCredentialsFromEnv, Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, OperationParams, } = require("ydb-sdk");
|
|
6
6
|
export { Types, TypedValues, TableDescription, Column, ExecuteQuerySettings };
|
|
7
7
|
export function createExecuteQuerySettings(options) {
|
|
8
8
|
const { keepInCache = true, idempotent = true } = options ?? {};
|
|
@@ -15,16 +15,46 @@ export function createExecuteQuerySettings(options) {
|
|
|
15
15
|
}
|
|
16
16
|
return settings;
|
|
17
17
|
}
|
|
18
|
+
export function createExecuteQuerySettingsWithTimeout(options) {
|
|
19
|
+
const settings = createExecuteQuerySettings(options);
|
|
20
|
+
const op = new OperationParams();
|
|
21
|
+
const seconds = Math.max(1, Math.ceil(options.timeoutMs / 1000));
|
|
22
|
+
// Limit both overall operation processing time and cancellation time on the
|
|
23
|
+
// server side so the probe fails fast instead of hanging for the default.
|
|
24
|
+
op.withOperationTimeoutSeconds(seconds);
|
|
25
|
+
op.withCancelAfterSeconds(seconds);
|
|
26
|
+
settings.withOperationParams(op);
|
|
27
|
+
return settings;
|
|
28
|
+
}
|
|
18
29
|
const DRIVER_READY_TIMEOUT_MS = 15000;
|
|
19
30
|
const TABLE_SESSION_TIMEOUT_MS = 20000;
|
|
20
31
|
const YDB_HEALTHCHECK_READY_TIMEOUT_MS = 5000;
|
|
21
32
|
const DRIVER_REFRESH_COOLDOWN_MS = 30000;
|
|
33
|
+
const STARTUP_PROBE_SESSION_TIMEOUT_MS = 3000;
|
|
22
34
|
let overrideConfig;
|
|
23
35
|
let driver;
|
|
24
36
|
let lastDriverRefreshAt = 0;
|
|
25
37
|
let driverRefreshInFlight = null;
|
|
26
38
|
// Test-only: allows injecting a mock Driver factory
|
|
27
39
|
let driverFactoryOverride;
|
|
40
|
+
export function isCompilationTimeoutError(error) {
|
|
41
|
+
if (!(error instanceof Error)) {
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
const msg = error.message ?? "";
|
|
45
|
+
if (/Timeout \(code 400090\)/i.test(msg) ||
|
|
46
|
+
/Query compilation timed out/i.test(msg)) {
|
|
47
|
+
return true;
|
|
48
|
+
}
|
|
49
|
+
// Startup probe uses explicit cancel-after; YDB returns Cancelled with
|
|
50
|
+
// issues like "Cancelling after 3000ms during compilation". Treat this as
|
|
51
|
+
// a compilation-time failure for fatal startup handling.
|
|
52
|
+
if (/Cancelled \(code 400160\)/i.test(msg) &&
|
|
53
|
+
/during compilation/i.test(msg)) {
|
|
54
|
+
return true;
|
|
55
|
+
}
|
|
56
|
+
return false;
|
|
57
|
+
}
|
|
28
58
|
function shouldTriggerDriverRefresh(error) {
|
|
29
59
|
if (!(error instanceof Error)) {
|
|
30
60
|
return false;
|
|
@@ -39,6 +69,12 @@ function shouldTriggerDriverRefresh(error) {
|
|
|
39
69
|
if (/SessionExpired|SESSION_EXPIRED|session.*expired/i.test(msg)) {
|
|
40
70
|
return true;
|
|
41
71
|
}
|
|
72
|
+
// YDB query compilation timeout (TIMEOUT code 400090) – treat as a signal
|
|
73
|
+
// to refresh the driver/session pool so that subsequent attempts use a
|
|
74
|
+
// fresh connection state.
|
|
75
|
+
if (isCompilationTimeoutError(error)) {
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
42
78
|
return false;
|
|
43
79
|
}
|
|
44
80
|
async function maybeRefreshDriverOnSessionError(error) {
|
|
@@ -55,7 +91,8 @@ async function maybeRefreshDriverOnSessionError(error) {
|
|
|
55
91
|
return;
|
|
56
92
|
}
|
|
57
93
|
lastDriverRefreshAt = now;
|
|
58
|
-
|
|
94
|
+
const errorMessage = error instanceof Error ? error.message ?? "" : String(error);
|
|
95
|
+
logger.warn({ err: error, errorMessage, lastDriverRefreshAt }, "YDB session-related error detected; refreshing driver");
|
|
59
96
|
try {
|
|
60
97
|
const refreshPromise = refreshDriver();
|
|
61
98
|
driverRefreshInFlight = refreshPromise;
|
|
@@ -131,6 +168,16 @@ export async function withSession(fn) {
|
|
|
131
168
|
throw err;
|
|
132
169
|
}
|
|
133
170
|
}
|
|
171
|
+
export async function withStartupProbeSession(fn) {
|
|
172
|
+
const d = getOrCreateDriver();
|
|
173
|
+
try {
|
|
174
|
+
return await d.tableClient.withSession(fn, STARTUP_PROBE_SESSION_TIMEOUT_MS);
|
|
175
|
+
}
|
|
176
|
+
catch (err) {
|
|
177
|
+
void maybeRefreshDriverOnSessionError(err);
|
|
178
|
+
throw err;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
134
181
|
export async function isYdbAvailable(timeoutMs = YDB_HEALTHCHECK_READY_TIMEOUT_MS) {
|
|
135
182
|
const d = getOrCreateDriver();
|
|
136
183
|
try {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ydb-qdrant",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "5.0.0",
|
|
4
4
|
"main": "dist/package/api.js",
|
|
5
5
|
"types": "dist/package/api.d.ts",
|
|
6
6
|
"exports": {
|
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
"scripts": {
|
|
17
17
|
"test": "vitest run --exclude \"test/integration/**\"",
|
|
18
18
|
"test:coverage": "vitest run --coverage --exclude \"test/integration/**\"",
|
|
19
|
-
"test:integration": "
|
|
20
|
-
"test:recall": "
|
|
19
|
+
"test:integration": "vitest run test/integration/YdbRealIntegration.test.ts && YDB_QDRANT_SEARCH_MODE=approximate vitest run test/integration/YdbRealIntegration.one-table.test.ts && YDB_QDRANT_SEARCH_MODE=exact vitest run test/integration/YdbRealIntegration.one-table.test.ts",
|
|
20
|
+
"test:recall": "YDB_QDRANT_SEARCH_MODE=approximate vitest run test/integration/YdbRealIntegration.one-table.test.ts && YDB_QDRANT_SEARCH_MODE=exact vitest run test/integration/YdbRealIntegration.one-table.test.ts",
|
|
21
21
|
"load:soak": "k6 run loadtest/soak-test.js",
|
|
22
22
|
"load:stress": "k6 run loadtest/stress-test.js",
|
|
23
23
|
"build": "tsc -p tsconfig.json",
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
],
|
|
50
50
|
"author": "",
|
|
51
51
|
"license": "Apache-2.0",
|
|
52
|
-
"description": "Qdrant-compatible Node.js/TypeScript API that stores/searches embeddings in YDB using
|
|
52
|
+
"description": "Qdrant-compatible Node.js/TypeScript API that stores/searches embeddings in YDB using a global one-table layout with exact and approximate KNN search over serialized vectors.",
|
|
53
53
|
"type": "module",
|
|
54
54
|
"publishConfig": {
|
|
55
55
|
"access": "public"
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
import type { DistanceKind, VectorType } from "../types.js";
|
|
2
|
-
export declare function notifyUpsert(tableName: string, count?: number): void;
|
|
3
|
-
export declare function requestIndexBuild(tableName: string, dimension: number, distance: DistanceKind, vectorType: VectorType, opts?: {
|
|
4
|
-
force?: boolean;
|
|
5
|
-
}): void;
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import { GLOBAL_POINTS_TABLE } from "../ydb/schema.js";
|
|
2
|
-
import { state, requestIndexBuildMultiTable, } from "./IndexScheduler.multi-table.js";
|
|
3
|
-
import { requestIndexBuildOneTable } from "./IndexScheduler.one-table.js";
|
|
4
|
-
export function notifyUpsert(tableName, count = 1) {
|
|
5
|
-
const now = Date.now();
|
|
6
|
-
const s = state[tableName] ?? {
|
|
7
|
-
lastUpsertMs: 0,
|
|
8
|
-
pending: false,
|
|
9
|
-
pointsUpserted: 0,
|
|
10
|
-
};
|
|
11
|
-
s.lastUpsertMs = now;
|
|
12
|
-
s.pointsUpserted += count;
|
|
13
|
-
state[tableName] = s;
|
|
14
|
-
}
|
|
15
|
-
export function requestIndexBuild(tableName, dimension, distance, vectorType, opts) {
|
|
16
|
-
if (tableName === GLOBAL_POINTS_TABLE) {
|
|
17
|
-
requestIndexBuildOneTable(tableName);
|
|
18
|
-
return;
|
|
19
|
-
}
|
|
20
|
-
requestIndexBuildMultiTable(tableName, dimension, distance, vectorType, opts);
|
|
21
|
-
}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import type { DistanceKind, VectorType } from "../types.js";
|
|
2
|
-
type CollectionKey = string;
|
|
3
|
-
export declare const state: Record<CollectionKey, {
|
|
4
|
-
lastUpsertMs: number;
|
|
5
|
-
timer?: NodeJS.Timeout;
|
|
6
|
-
pending: boolean;
|
|
7
|
-
pointsUpserted: number;
|
|
8
|
-
}>;
|
|
9
|
-
export declare function requestIndexBuildMultiTable(tableName: string, dimension: number, distance: DistanceKind, vectorType: VectorType, opts?: {
|
|
10
|
-
force?: boolean;
|
|
11
|
-
}): void;
|
|
12
|
-
export {};
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
import { buildVectorIndex } from "../repositories/collectionsRepo.js";
|
|
2
|
-
import { logger } from "../logging/logger.js";
|
|
3
|
-
const QUIET_MS = 10000;
|
|
4
|
-
const MIN_POINTS_THRESHOLD = 100;
|
|
5
|
-
export const state = {};
|
|
6
|
-
export function requestIndexBuildMultiTable(tableName, dimension, distance, vectorType, opts) {
|
|
7
|
-
const s = state[tableName] ?? {
|
|
8
|
-
lastUpsertMs: 0,
|
|
9
|
-
pending: false,
|
|
10
|
-
pointsUpserted: 0,
|
|
11
|
-
};
|
|
12
|
-
state[tableName] = s;
|
|
13
|
-
if (opts?.force) {
|
|
14
|
-
logger.info({ tableName }, "index build (force) starting");
|
|
15
|
-
void buildVectorIndex(tableName, dimension, distance, vectorType)
|
|
16
|
-
.then(() => {
|
|
17
|
-
logger.info({ tableName }, "index build (force) completed");
|
|
18
|
-
s.pointsUpserted = 0;
|
|
19
|
-
})
|
|
20
|
-
.catch((err) => {
|
|
21
|
-
logger.error({ err, tableName }, "index build (force) failed");
|
|
22
|
-
});
|
|
23
|
-
return;
|
|
24
|
-
}
|
|
25
|
-
if (s.pending && s.timer) {
|
|
26
|
-
return;
|
|
27
|
-
}
|
|
28
|
-
s.pending = true;
|
|
29
|
-
s.timer = setTimeout(function tryBuild() {
|
|
30
|
-
const since = Date.now() - (state[tableName]?.lastUpsertMs ?? 0);
|
|
31
|
-
if (since < QUIET_MS) {
|
|
32
|
-
s.timer = setTimeout(tryBuild, QUIET_MS - since);
|
|
33
|
-
return;
|
|
34
|
-
}
|
|
35
|
-
const pointsCount = state[tableName]?.pointsUpserted ?? 0;
|
|
36
|
-
if (pointsCount < MIN_POINTS_THRESHOLD) {
|
|
37
|
-
logger.info({ tableName, pointsCount, threshold: MIN_POINTS_THRESHOLD }, "index build skipped (below threshold)");
|
|
38
|
-
s.pending = false;
|
|
39
|
-
s.timer = undefined;
|
|
40
|
-
return;
|
|
41
|
-
}
|
|
42
|
-
logger.info({ tableName, pointsCount }, "index build (scheduled) starting");
|
|
43
|
-
void buildVectorIndex(tableName, dimension, distance, vectorType)
|
|
44
|
-
.then(() => {
|
|
45
|
-
logger.info({ tableName }, "index build (scheduled) completed");
|
|
46
|
-
state[tableName].pointsUpserted = 0;
|
|
47
|
-
})
|
|
48
|
-
.catch((err) => logger.error({ err, tableName }, "index build (scheduled) failed"))
|
|
49
|
-
.finally(() => {
|
|
50
|
-
s.pending = false;
|
|
51
|
-
s.timer = undefined;
|
|
52
|
-
});
|
|
53
|
-
}, QUIET_MS);
|
|
54
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function requestIndexBuildOneTable(tableName: string): void;
|
|
@@ -1,3 +0,0 @@
|
|
|
1
|
-
import type { DistanceKind, VectorType } from "../types";
|
|
2
|
-
export declare function createCollectionMultiTable(metaKey: string, dim: number, distance: DistanceKind, vectorType: VectorType, tableName: string): Promise<void>;
|
|
3
|
-
export declare function deleteCollectionMultiTable(metaKey: string, tableName: string): Promise<void>;
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import { Types, TypedValues, withSession, TableDescription, Column, createExecuteQuerySettings, } from "../ydb/client.js";
|
|
2
|
-
import { upsertCollectionMeta } from "./collectionsRepo.shared.js";
|
|
3
|
-
export async function createCollectionMultiTable(metaKey, dim, distance, vectorType, tableName) {
|
|
4
|
-
await withSession(async (s) => {
|
|
5
|
-
const desc = new TableDescription()
|
|
6
|
-
.withColumns(new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
|
|
7
|
-
.withPrimaryKey("point_id");
|
|
8
|
-
await s.createTable(tableName, desc);
|
|
9
|
-
});
|
|
10
|
-
await upsertCollectionMeta(metaKey, dim, distance, vectorType, tableName);
|
|
11
|
-
}
|
|
12
|
-
export async function deleteCollectionMultiTable(metaKey, tableName) {
|
|
13
|
-
await withSession(async (s) => {
|
|
14
|
-
await s.dropTable(tableName);
|
|
15
|
-
});
|
|
16
|
-
const delMeta = `
|
|
17
|
-
DECLARE $collection AS Utf8;
|
|
18
|
-
DELETE FROM qdr__collections WHERE collection = $collection;
|
|
19
|
-
`;
|
|
20
|
-
await withSession(async (s) => {
|
|
21
|
-
const settings = createExecuteQuerySettings();
|
|
22
|
-
await s.executeQuery(delMeta, { $collection: TypedValues.utf8(metaKey) }, undefined, settings);
|
|
23
|
-
});
|
|
24
|
-
}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import type { DistanceKind } from "../types";
|
|
2
|
-
export declare function upsertPointsMultiTable(tableName: string, points: Array<{
|
|
3
|
-
id: string | number;
|
|
4
|
-
vector: number[];
|
|
5
|
-
payload?: Record<string, unknown>;
|
|
6
|
-
}>, dimension: number): Promise<number>;
|
|
7
|
-
export declare function searchPointsMultiTable(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number): Promise<Array<{
|
|
8
|
-
id: string;
|
|
9
|
-
score: number;
|
|
10
|
-
payload?: Record<string, unknown>;
|
|
11
|
-
}>>;
|
|
12
|
-
export declare function deletePointsMultiTable(tableName: string, ids: Array<string | number>): Promise<number>;
|
|
@@ -1,147 +0,0 @@
|
|
|
1
|
-
import { Types, TypedValues, withSession, createExecuteQuerySettings, } from "../ydb/client.js";
|
|
2
|
-
import { buildVectorParam } from "../ydb/helpers.js";
|
|
3
|
-
import { logger } from "../logging/logger.js";
|
|
4
|
-
import { notifyUpsert } from "../indexing/IndexScheduler.js";
|
|
5
|
-
import { VECTOR_INDEX_BUILD_ENABLED } from "../config/env.js";
|
|
6
|
-
import { mapDistanceToKnnFn } from "../utils/distance.js";
|
|
7
|
-
import { withRetry, isTransientYdbError } from "../utils/retry.js";
|
|
8
|
-
import { UPSERT_BATCH_SIZE } from "../ydb/schema.js";
|
|
9
|
-
export async function upsertPointsMultiTable(tableName, points, dimension) {
|
|
10
|
-
for (const p of points) {
|
|
11
|
-
const id = String(p.id);
|
|
12
|
-
if (p.vector.length !== dimension) {
|
|
13
|
-
throw new Error(`Vector dimension mismatch for id=${id}: got ${p.vector.length}, expected ${dimension}`);
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
let upserted = 0;
|
|
17
|
-
await withSession(async (s) => {
|
|
18
|
-
const settings = createExecuteQuerySettings();
|
|
19
|
-
for (let i = 0; i < points.length; i += UPSERT_BATCH_SIZE) {
|
|
20
|
-
const batch = points.slice(i, i + UPSERT_BATCH_SIZE);
|
|
21
|
-
const ddl = `
|
|
22
|
-
DECLARE $rows AS List<Struct<
|
|
23
|
-
point_id: Utf8,
|
|
24
|
-
vec: List<Float>,
|
|
25
|
-
payload: JsonDocument
|
|
26
|
-
>>;
|
|
27
|
-
|
|
28
|
-
UPSERT INTO ${tableName} (point_id, embedding, payload)
|
|
29
|
-
SELECT
|
|
30
|
-
point_id,
|
|
31
|
-
Untag(Knn::ToBinaryStringFloat(vec), "FloatVector") AS embedding,
|
|
32
|
-
payload
|
|
33
|
-
FROM AS_TABLE($rows);
|
|
34
|
-
`;
|
|
35
|
-
const rowType = Types.struct({
|
|
36
|
-
point_id: Types.UTF8,
|
|
37
|
-
vec: Types.list(Types.FLOAT),
|
|
38
|
-
payload: Types.JSON_DOCUMENT,
|
|
39
|
-
});
|
|
40
|
-
const rowsValue = TypedValues.list(rowType, batch.map((p) => ({
|
|
41
|
-
point_id: String(p.id),
|
|
42
|
-
vec: p.vector,
|
|
43
|
-
payload: JSON.stringify(p.payload ?? {}),
|
|
44
|
-
})));
|
|
45
|
-
const params = {
|
|
46
|
-
$rows: rowsValue,
|
|
47
|
-
};
|
|
48
|
-
await withRetry(() => s.executeQuery(ddl, params, undefined, settings), {
|
|
49
|
-
isTransient: isTransientYdbError,
|
|
50
|
-
context: { tableName, batchSize: batch.length },
|
|
51
|
-
});
|
|
52
|
-
upserted += batch.length;
|
|
53
|
-
}
|
|
54
|
-
});
|
|
55
|
-
notifyUpsert(tableName, upserted);
|
|
56
|
-
return upserted;
|
|
57
|
-
}
|
|
58
|
-
export async function searchPointsMultiTable(tableName, queryVector, top, withPayload, distance, dimension) {
|
|
59
|
-
if (queryVector.length !== dimension) {
|
|
60
|
-
throw new Error(`Vector dimension mismatch: got ${queryVector.length}, expected ${dimension}`);
|
|
61
|
-
}
|
|
62
|
-
const { fn, order } = mapDistanceToKnnFn(distance);
|
|
63
|
-
const qf = buildVectorParam(queryVector);
|
|
64
|
-
const params = {
|
|
65
|
-
$qf: qf,
|
|
66
|
-
$k2: TypedValues.uint32(top),
|
|
67
|
-
};
|
|
68
|
-
const settings = createExecuteQuerySettings();
|
|
69
|
-
const buildQuery = (useIndex) => `
|
|
70
|
-
DECLARE $qf AS List<Float>;
|
|
71
|
-
DECLARE $k2 AS Uint32;
|
|
72
|
-
$qbinf = Knn::ToBinaryStringFloat($qf);
|
|
73
|
-
SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
|
|
74
|
-
FROM ${tableName}${useIndex ? " VIEW emb_idx" : ""}
|
|
75
|
-
ORDER BY score ${order}
|
|
76
|
-
LIMIT $k2;
|
|
77
|
-
`;
|
|
78
|
-
let rs;
|
|
79
|
-
if (VECTOR_INDEX_BUILD_ENABLED) {
|
|
80
|
-
try {
|
|
81
|
-
rs = await withSession(async (s) => {
|
|
82
|
-
return await s.executeQuery(buildQuery(true), params, undefined, settings);
|
|
83
|
-
});
|
|
84
|
-
logger.info({ tableName }, "vector index found; using index for search");
|
|
85
|
-
}
|
|
86
|
-
catch (e) {
|
|
87
|
-
const msg = e instanceof Error ? e.message : String(e);
|
|
88
|
-
const indexUnavailable = /not found|does not exist|no such index|no global index|is not ready to use/i.test(msg);
|
|
89
|
-
if (indexUnavailable) {
|
|
90
|
-
logger.info({ tableName }, "vector index not available (missing or building); falling back to table scan");
|
|
91
|
-
rs = await withSession(async (s) => {
|
|
92
|
-
return await s.executeQuery(buildQuery(false), params, undefined, settings);
|
|
93
|
-
});
|
|
94
|
-
}
|
|
95
|
-
else {
|
|
96
|
-
throw e;
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
else {
|
|
101
|
-
rs = await withSession(async (s) => {
|
|
102
|
-
return await s.executeQuery(buildQuery(false), params, undefined, settings);
|
|
103
|
-
});
|
|
104
|
-
}
|
|
105
|
-
const rowset = rs.resultSets?.[0];
|
|
106
|
-
const rows = (rowset?.rows ?? []);
|
|
107
|
-
return rows.map((row) => {
|
|
108
|
-
const id = row.items?.[0]?.textValue;
|
|
109
|
-
if (typeof id !== "string") {
|
|
110
|
-
throw new Error("point_id is missing in YDB search result");
|
|
111
|
-
}
|
|
112
|
-
let payload;
|
|
113
|
-
let scoreIdx = 1;
|
|
114
|
-
if (withPayload) {
|
|
115
|
-
const payloadText = row.items?.[1]?.textValue;
|
|
116
|
-
if (payloadText) {
|
|
117
|
-
try {
|
|
118
|
-
payload = JSON.parse(payloadText);
|
|
119
|
-
}
|
|
120
|
-
catch {
|
|
121
|
-
payload = undefined;
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
scoreIdx = 2;
|
|
125
|
-
}
|
|
126
|
-
const score = Number(row.items?.[scoreIdx]?.floatValue ?? row.items?.[scoreIdx]?.textValue);
|
|
127
|
-
return { id, score, ...(payload ? { payload } : {}) };
|
|
128
|
-
});
|
|
129
|
-
}
|
|
130
|
-
export async function deletePointsMultiTable(tableName, ids) {
|
|
131
|
-
let deleted = 0;
|
|
132
|
-
await withSession(async (s) => {
|
|
133
|
-
const settings = createExecuteQuerySettings();
|
|
134
|
-
for (const id of ids) {
|
|
135
|
-
const yql = `
|
|
136
|
-
DECLARE $id AS Utf8;
|
|
137
|
-
DELETE FROM ${tableName} WHERE point_id = $id;
|
|
138
|
-
`;
|
|
139
|
-
const params = {
|
|
140
|
-
$id: TypedValues.utf8(String(id)),
|
|
141
|
-
};
|
|
142
|
-
await s.executeQuery(yql, params, undefined, settings);
|
|
143
|
-
deleted += 1;
|
|
144
|
-
}
|
|
145
|
-
});
|
|
146
|
-
return deleted;
|
|
147
|
-
}
|