ydb-qdrant 3.0.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/SmokeTest.js +1 -1
- package/dist/config/env.d.ts +6 -0
- package/dist/config/env.js +17 -0
- package/dist/index.js +5 -2
- package/dist/indexing/IndexScheduler.js +6 -51
- package/dist/indexing/IndexScheduler.multi-table.d.ts +12 -0
- package/dist/indexing/IndexScheduler.multi-table.js +54 -0
- package/dist/indexing/IndexScheduler.one-table.d.ts +1 -0
- package/dist/indexing/IndexScheduler.one-table.js +4 -0
- package/dist/package/{Api.d.ts → api.d.ts} +3 -2
- package/dist/package/api.js +55 -0
- package/dist/repositories/collectionsRepo.d.ts +3 -2
- package/dist/repositories/collectionsRepo.js +27 -53
- package/dist/repositories/collectionsRepo.multi-table.d.ts +3 -0
- package/dist/repositories/collectionsRepo.multi-table.js +23 -0
- package/dist/repositories/collectionsRepo.one-table.d.ts +3 -0
- package/dist/repositories/collectionsRepo.one-table.js +25 -0
- package/dist/repositories/collectionsRepo.shared.d.ts +2 -0
- package/dist/repositories/collectionsRepo.shared.js +23 -0
- package/dist/repositories/pointsRepo.d.ts +3 -3
- package/dist/repositories/pointsRepo.js +15 -158
- package/dist/repositories/pointsRepo.multi-table.d.ts +12 -0
- package/dist/repositories/pointsRepo.multi-table.js +129 -0
- package/dist/repositories/pointsRepo.one-table.d.ts +12 -0
- package/dist/repositories/pointsRepo.one-table.js +111 -0
- package/dist/routes/collections.js +2 -1
- package/dist/routes/points.js +2 -1
- package/dist/services/CollectionService.d.ts +35 -0
- package/dist/services/CollectionService.js +85 -0
- package/dist/services/CollectionService.multi-table.d.ts +5 -0
- package/dist/services/CollectionService.multi-table.js +7 -0
- package/dist/services/CollectionService.one-table.d.ts +5 -0
- package/dist/services/CollectionService.one-table.js +9 -0
- package/dist/services/CollectionService.shared.d.ts +11 -0
- package/dist/services/CollectionService.shared.js +17 -0
- package/dist/services/PointsService.d.ts +23 -0
- package/dist/services/PointsService.js +128 -0
- package/dist/services/errors.d.ts +9 -0
- package/dist/services/errors.js +9 -0
- package/dist/utils/distance.d.ts +6 -0
- package/dist/utils/distance.js +28 -0
- package/dist/utils/normalization.d.ts +10 -0
- package/dist/utils/normalization.js +92 -0
- package/dist/utils/retry.d.ts +8 -0
- package/dist/utils/retry.js +28 -0
- package/dist/utils/tenant.d.ts +1 -0
- package/dist/utils/tenant.js +3 -0
- package/dist/ydb/schema.d.ts +2 -0
- package/dist/ydb/schema.js +27 -0
- package/package.json +4 -4
- package/dist/package/Api.js +0 -79
- package/dist/services/QdrantService.d.ts +0 -54
- package/dist/services/QdrantService.js +0 -313
package/README.md
CHANGED
|
@@ -75,6 +75,8 @@ Optional env:
|
|
|
75
75
|
# Server
|
|
76
76
|
export PORT=8080
|
|
77
77
|
export LOG_LEVEL=info
|
|
78
|
+
# Collection storage mode (optional; default is multi_table)
|
|
79
|
+
export YDB_QDRANT_COLLECTION_STORAGE_MODE=multi_table # or one_table
|
|
78
80
|
```
|
|
79
81
|
|
|
80
82
|
## Use as a Node.js library (npm package)
|
|
@@ -398,11 +400,13 @@ curl -X POST http://localhost:8080/collections/mycol/points/delete \
|
|
|
398
400
|
```
|
|
399
401
|
|
|
400
402
|
## Notes
|
|
401
|
-
-
|
|
402
|
-
-
|
|
403
|
+
- Storage layout:
|
|
404
|
+
- **multi_table** (default): one YDB table per collection; metadata is tracked in `qdr__collections`.
|
|
405
|
+
- **one_table**: a single global table `qdrant_all_points` with `(uid, point_id)` PK, where `uid` encodes tenant+collection.
|
|
406
|
+
- Per‑collection table schema (multi_table): `point_id Utf8` (PK), `embedding String` (binary), `payload JsonDocument`.
|
|
403
407
|
- Vectors are serialized with `Knn::ToBinaryStringFloat`.
|
|
404
408
|
- Search uses a single-phase top‑k over `embedding` with automatic YDB vector index (`emb_idx`) when available; falls back to table scan if missing.
|
|
405
|
-
- **Vector index auto-build
|
|
409
|
+
- **Vector index auto-build** (multi_table mode only): After ≥100 points upserted + 5s quiet window, a `vector_kmeans_tree` index (levels=1, clusters=128) is built automatically. Incremental updates (<100 points) skip index rebuild. In one_table mode, vector indexes are not supported and all searches use table scans.
|
|
406
410
|
- **Concurrency**: During index rebuilds, YDB may return transient `Aborted`/schema metadata errors. Upserts include bounded retries with backoff to handle this automatically.
|
|
407
411
|
- Filters are not yet modeled; can be added if needed.
|
|
408
412
|
|
package/dist/SmokeTest.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import "dotenv/config";
|
|
2
|
-
import { createYdbQdrantClient } from "./package/
|
|
2
|
+
import { createYdbQdrantClient } from "./package/api.js";
|
|
3
3
|
async function main() {
|
|
4
4
|
const tenant = process.env.SMOKE_TENANT ?? "smoke";
|
|
5
5
|
const collection = process.env.SMOKE_COLLECTION ?? "demo";
|
package/dist/config/env.d.ts
CHANGED
|
@@ -4,3 +4,9 @@ export declare const YDB_DATABASE: string;
|
|
|
4
4
|
export declare const PORT: number;
|
|
5
5
|
export declare const LOG_LEVEL: string;
|
|
6
6
|
export declare const VECTOR_INDEX_BUILD_ENABLED: boolean;
|
|
7
|
+
export declare enum CollectionStorageMode {
|
|
8
|
+
MultiTable = "multi_table",
|
|
9
|
+
OneTable = "one_table"
|
|
10
|
+
}
|
|
11
|
+
export declare const COLLECTION_STORAGE_MODE: CollectionStorageMode;
|
|
12
|
+
export declare function isOneTableMode(mode: CollectionStorageMode): mode is CollectionStorageMode.OneTable;
|
package/dist/config/env.js
CHANGED
|
@@ -18,3 +18,20 @@ function parseBooleanEnv(value, defaultValue) {
|
|
|
18
18
|
return true;
|
|
19
19
|
}
|
|
20
20
|
export const VECTOR_INDEX_BUILD_ENABLED = parseBooleanEnv(process.env.VECTOR_INDEX_BUILD_ENABLED, false);
|
|
21
|
+
export var CollectionStorageMode;
|
|
22
|
+
(function (CollectionStorageMode) {
|
|
23
|
+
CollectionStorageMode["MultiTable"] = "multi_table";
|
|
24
|
+
CollectionStorageMode["OneTable"] = "one_table";
|
|
25
|
+
})(CollectionStorageMode || (CollectionStorageMode = {}));
|
|
26
|
+
function resolveCollectionStorageModeEnv() {
|
|
27
|
+
const explicit = process.env.YDB_QDRANT_COLLECTION_STORAGE_MODE ??
|
|
28
|
+
process.env.YDB_QDRANT_TABLE_LAYOUT;
|
|
29
|
+
if (explicit?.trim().toLowerCase() === CollectionStorageMode.OneTable) {
|
|
30
|
+
return CollectionStorageMode.OneTable;
|
|
31
|
+
}
|
|
32
|
+
return CollectionStorageMode.MultiTable;
|
|
33
|
+
}
|
|
34
|
+
export const COLLECTION_STORAGE_MODE = resolveCollectionStorageModeEnv();
|
|
35
|
+
export function isOneTableMode(mode) {
|
|
36
|
+
return mode === CollectionStorageMode.OneTable;
|
|
37
|
+
}
|
package/dist/index.js
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import "dotenv/config";
|
|
2
2
|
import { buildServer } from "./server.js";
|
|
3
|
-
import { PORT } from "./config/env.js";
|
|
3
|
+
import { PORT, COLLECTION_STORAGE_MODE, isOneTableMode, } from "./config/env.js";
|
|
4
4
|
import { logger } from "./logging/logger.js";
|
|
5
5
|
import { readyOrThrow } from "./ydb/client.js";
|
|
6
|
-
import { ensureMetaTable } from "./ydb/schema.js";
|
|
6
|
+
import { ensureMetaTable, ensureGlobalPointsTable } from "./ydb/schema.js";
|
|
7
7
|
async function start() {
|
|
8
8
|
try {
|
|
9
9
|
await readyOrThrow();
|
|
10
10
|
await ensureMetaTable();
|
|
11
|
+
if (isOneTableMode(COLLECTION_STORAGE_MODE)) {
|
|
12
|
+
await ensureGlobalPointsTable();
|
|
13
|
+
}
|
|
11
14
|
}
|
|
12
15
|
catch (err) {
|
|
13
16
|
logger.error({ err }, "YDB not ready; startup continues, requests may fail until configured.");
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
const MIN_POINTS_THRESHOLD = 100; // only rebuild if at least this many points upserted
|
|
5
|
-
const state = {};
|
|
1
|
+
import { GLOBAL_POINTS_TABLE } from "../ydb/schema.js";
|
|
2
|
+
import { state, requestIndexBuildMultiTable, } from "./IndexScheduler.multi-table.js";
|
|
3
|
+
import { requestIndexBuildOneTable } from "./IndexScheduler.one-table.js";
|
|
6
4
|
export function notifyUpsert(tableName, count = 1) {
|
|
7
5
|
const now = Date.now();
|
|
8
6
|
const s = state[tableName] ?? {
|
|
@@ -15,52 +13,9 @@ export function notifyUpsert(tableName, count = 1) {
|
|
|
15
13
|
state[tableName] = s;
|
|
16
14
|
}
|
|
17
15
|
export function requestIndexBuild(tableName, dimension, distance, vectorType, opts) {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
pending: false,
|
|
21
|
-
pointsUpserted: 0,
|
|
22
|
-
};
|
|
23
|
-
state[tableName] = s;
|
|
24
|
-
if (opts?.force) {
|
|
25
|
-
logger.info({ tableName }, "index build (force) starting");
|
|
26
|
-
void buildVectorIndex(tableName, dimension, distance, vectorType)
|
|
27
|
-
.then(() => {
|
|
28
|
-
logger.info({ tableName }, "index build (force) completed");
|
|
29
|
-
s.pointsUpserted = 0;
|
|
30
|
-
})
|
|
31
|
-
.catch((err) => {
|
|
32
|
-
logger.error({ err, tableName }, "index build (force) failed");
|
|
33
|
-
});
|
|
34
|
-
return;
|
|
35
|
-
}
|
|
36
|
-
if (s.pending && s.timer) {
|
|
37
|
-
// already scheduled; timer will check quiet window
|
|
16
|
+
if (tableName === GLOBAL_POINTS_TABLE) {
|
|
17
|
+
requestIndexBuildOneTable(tableName);
|
|
38
18
|
return;
|
|
39
19
|
}
|
|
40
|
-
|
|
41
|
-
s.timer = setTimeout(function tryBuild() {
|
|
42
|
-
const since = Date.now() - (state[tableName]?.lastUpsertMs ?? 0);
|
|
43
|
-
if (since < QUIET_MS) {
|
|
44
|
-
s.timer = setTimeout(tryBuild, QUIET_MS - since);
|
|
45
|
-
return;
|
|
46
|
-
}
|
|
47
|
-
const pointsCount = state[tableName]?.pointsUpserted ?? 0;
|
|
48
|
-
if (pointsCount < MIN_POINTS_THRESHOLD) {
|
|
49
|
-
logger.info({ tableName, pointsCount, threshold: MIN_POINTS_THRESHOLD }, "index build skipped (below threshold)");
|
|
50
|
-
s.pending = false;
|
|
51
|
-
s.timer = undefined;
|
|
52
|
-
return;
|
|
53
|
-
}
|
|
54
|
-
logger.info({ tableName, pointsCount }, "index build (scheduled) starting");
|
|
55
|
-
void buildVectorIndex(tableName, dimension, distance, vectorType)
|
|
56
|
-
.then(() => {
|
|
57
|
-
logger.info({ tableName }, "index build (scheduled) completed");
|
|
58
|
-
state[tableName].pointsUpserted = 0;
|
|
59
|
-
})
|
|
60
|
-
.catch((err) => logger.error({ err, tableName }, "index build (scheduled) failed"))
|
|
61
|
-
.finally(() => {
|
|
62
|
-
s.pending = false;
|
|
63
|
-
s.timer = undefined;
|
|
64
|
-
});
|
|
65
|
-
}, QUIET_MS);
|
|
20
|
+
requestIndexBuildMultiTable(tableName, dimension, distance, vectorType, opts);
|
|
66
21
|
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { DistanceKind, VectorType } from "../types.js";
|
|
2
|
+
type CollectionKey = string;
|
|
3
|
+
export declare const state: Record<CollectionKey, {
|
|
4
|
+
lastUpsertMs: number;
|
|
5
|
+
timer?: NodeJS.Timeout;
|
|
6
|
+
pending: boolean;
|
|
7
|
+
pointsUpserted: number;
|
|
8
|
+
}>;
|
|
9
|
+
export declare function requestIndexBuildMultiTable(tableName: string, dimension: number, distance: DistanceKind, vectorType: VectorType, opts?: {
|
|
10
|
+
force?: boolean;
|
|
11
|
+
}): void;
|
|
12
|
+
export {};
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { buildVectorIndex } from "../repositories/collectionsRepo.js";
|
|
2
|
+
import { logger } from "../logging/logger.js";
|
|
3
|
+
const QUIET_MS = 10000;
|
|
4
|
+
const MIN_POINTS_THRESHOLD = 100;
|
|
5
|
+
export const state = {};
|
|
6
|
+
export function requestIndexBuildMultiTable(tableName, dimension, distance, vectorType, opts) {
|
|
7
|
+
const s = state[tableName] ?? {
|
|
8
|
+
lastUpsertMs: 0,
|
|
9
|
+
pending: false,
|
|
10
|
+
pointsUpserted: 0,
|
|
11
|
+
};
|
|
12
|
+
state[tableName] = s;
|
|
13
|
+
if (opts?.force) {
|
|
14
|
+
logger.info({ tableName }, "index build (force) starting");
|
|
15
|
+
void buildVectorIndex(tableName, dimension, distance, vectorType)
|
|
16
|
+
.then(() => {
|
|
17
|
+
logger.info({ tableName }, "index build (force) completed");
|
|
18
|
+
s.pointsUpserted = 0;
|
|
19
|
+
})
|
|
20
|
+
.catch((err) => {
|
|
21
|
+
logger.error({ err, tableName }, "index build (force) failed");
|
|
22
|
+
});
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
if (s.pending && s.timer) {
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
s.pending = true;
|
|
29
|
+
s.timer = setTimeout(function tryBuild() {
|
|
30
|
+
const since = Date.now() - (state[tableName]?.lastUpsertMs ?? 0);
|
|
31
|
+
if (since < QUIET_MS) {
|
|
32
|
+
s.timer = setTimeout(tryBuild, QUIET_MS - since);
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
const pointsCount = state[tableName]?.pointsUpserted ?? 0;
|
|
36
|
+
if (pointsCount < MIN_POINTS_THRESHOLD) {
|
|
37
|
+
logger.info({ tableName, pointsCount, threshold: MIN_POINTS_THRESHOLD }, "index build skipped (below threshold)");
|
|
38
|
+
s.pending = false;
|
|
39
|
+
s.timer = undefined;
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
logger.info({ tableName, pointsCount }, "index build (scheduled) starting");
|
|
43
|
+
void buildVectorIndex(tableName, dimension, distance, vectorType)
|
|
44
|
+
.then(() => {
|
|
45
|
+
logger.info({ tableName }, "index build (scheduled) completed");
|
|
46
|
+
state[tableName].pointsUpserted = 0;
|
|
47
|
+
})
|
|
48
|
+
.catch((err) => logger.error({ err, tableName }, "index build (scheduled) failed"))
|
|
49
|
+
.finally(() => {
|
|
50
|
+
s.pending = false;
|
|
51
|
+
s.timer = undefined;
|
|
52
|
+
});
|
|
53
|
+
}, QUIET_MS);
|
|
54
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function requestIndexBuildOneTable(tableName: string): void;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { IAuthService } from "ydb-sdk";
|
|
2
|
-
import { createCollection as serviceCreateCollection, deleteCollection as serviceDeleteCollection, getCollection as serviceGetCollection, putCollectionIndex as servicePutCollectionIndex
|
|
3
|
-
|
|
2
|
+
import { createCollection as serviceCreateCollection, deleteCollection as serviceDeleteCollection, getCollection as serviceGetCollection, putCollectionIndex as servicePutCollectionIndex } from "../services/CollectionService.js";
|
|
3
|
+
import { upsertPoints as serviceUpsertPoints, searchPoints as serviceSearchPoints, deletePoints as serviceDeletePoints } from "../services/PointsService.js";
|
|
4
|
+
export { QdrantServiceError } from "../services/errors.js";
|
|
4
5
|
export { CreateCollectionReq, UpsertPointsReq, SearchReq, DeletePointsReq, } from "../types.js";
|
|
5
6
|
type CreateCollectionResult = Awaited<ReturnType<typeof serviceCreateCollection>>;
|
|
6
7
|
type GetCollectionResult = Awaited<ReturnType<typeof serviceGetCollection>>;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { readyOrThrow, configureDriver } from "../ydb/client.js";
|
|
2
|
+
import { ensureMetaTable } from "../ydb/schema.js";
|
|
3
|
+
import { createCollection as serviceCreateCollection, deleteCollection as serviceDeleteCollection, getCollection as serviceGetCollection, putCollectionIndex as servicePutCollectionIndex, } from "../services/CollectionService.js";
|
|
4
|
+
import { upsertPoints as serviceUpsertPoints, searchPoints as serviceSearchPoints, deletePoints as serviceDeletePoints, } from "../services/PointsService.js";
|
|
5
|
+
export { QdrantServiceError } from "../services/errors.js";
|
|
6
|
+
export { CreateCollectionReq, UpsertPointsReq, SearchReq, DeletePointsReq, } from "../types.js";
|
|
7
|
+
function buildTenantClient(resolveTenant) {
|
|
8
|
+
return {
|
|
9
|
+
createCollection(collection, body) {
|
|
10
|
+
return serviceCreateCollection({ tenant: resolveTenant(), collection }, body);
|
|
11
|
+
},
|
|
12
|
+
getCollection(collection) {
|
|
13
|
+
return serviceGetCollection({ tenant: resolveTenant(), collection });
|
|
14
|
+
},
|
|
15
|
+
deleteCollection(collection) {
|
|
16
|
+
return serviceDeleteCollection({ tenant: resolveTenant(), collection });
|
|
17
|
+
},
|
|
18
|
+
putCollectionIndex(collection) {
|
|
19
|
+
return servicePutCollectionIndex({ tenant: resolveTenant(), collection });
|
|
20
|
+
},
|
|
21
|
+
upsertPoints(collection, body) {
|
|
22
|
+
return serviceUpsertPoints({ tenant: resolveTenant(), collection }, body);
|
|
23
|
+
},
|
|
24
|
+
searchPoints(collection, body) {
|
|
25
|
+
return serviceSearchPoints({ tenant: resolveTenant(), collection }, body);
|
|
26
|
+
},
|
|
27
|
+
deletePoints(collection, body) {
|
|
28
|
+
return serviceDeletePoints({ tenant: resolveTenant(), collection }, body);
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
export async function createYdbQdrantClient(options = {}) {
|
|
33
|
+
if (options.endpoint !== undefined ||
|
|
34
|
+
options.database !== undefined ||
|
|
35
|
+
options.connectionString !== undefined ||
|
|
36
|
+
options.authService !== undefined) {
|
|
37
|
+
configureDriver({
|
|
38
|
+
endpoint: options.endpoint,
|
|
39
|
+
database: options.database,
|
|
40
|
+
connectionString: options.connectionString,
|
|
41
|
+
authService: options.authService,
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
await readyOrThrow();
|
|
45
|
+
await ensureMetaTable();
|
|
46
|
+
const defaultTenant = options.defaultTenant ?? "default";
|
|
47
|
+
const baseClient = buildTenantClient(() => defaultTenant);
|
|
48
|
+
const client = {
|
|
49
|
+
...baseClient,
|
|
50
|
+
forTenant(tenantId) {
|
|
51
|
+
return buildTenantClient(() => tenantId);
|
|
52
|
+
},
|
|
53
|
+
};
|
|
54
|
+
return client;
|
|
55
|
+
}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import type { DistanceKind, VectorType } from "../types";
|
|
2
|
-
|
|
2
|
+
import { type CollectionStorageMode } from "../config/env.js";
|
|
3
|
+
export declare function createCollection(metaKey: string, dim: number, distance: DistanceKind, vectorType: VectorType, tableName: string, layout?: CollectionStorageMode): Promise<void>;
|
|
3
4
|
export declare function getCollectionMeta(metaKey: string): Promise<{
|
|
4
5
|
table: string;
|
|
5
6
|
dimension: number;
|
|
6
7
|
distance: DistanceKind;
|
|
7
8
|
vectorType: VectorType;
|
|
8
9
|
} | null>;
|
|
9
|
-
export declare function deleteCollection(metaKey: string): Promise<void>;
|
|
10
|
+
export declare function deleteCollection(metaKey: string, uid?: string): Promise<void>;
|
|
10
11
|
export declare function buildVectorIndex(tableName: string, dimension: number, distance: DistanceKind, vectorType: VectorType): Promise<void>;
|
|
@@ -1,31 +1,16 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
DECLARE $vtype AS Utf8;
|
|
15
|
-
DECLARE $created AS Timestamp;
|
|
16
|
-
UPSERT INTO qdr__collections (collection, table_name, vector_dimension, distance, vector_type, created_at)
|
|
17
|
-
VALUES ($collection, $table, $dim, $distance, $vtype, $created);
|
|
18
|
-
`;
|
|
19
|
-
await withSession(async (s) => {
|
|
20
|
-
await s.executeQuery(upsertMeta, {
|
|
21
|
-
$collection: TypedValues.utf8(metaKey),
|
|
22
|
-
$table: TypedValues.utf8(tableName),
|
|
23
|
-
$dim: TypedValues.uint32(dim),
|
|
24
|
-
$distance: TypedValues.utf8(distance),
|
|
25
|
-
$vtype: TypedValues.utf8(vectorType),
|
|
26
|
-
$created: TypedValues.timestamp(new Date()),
|
|
27
|
-
});
|
|
28
|
-
});
|
|
1
|
+
import { TypedValues, withSession } from "../ydb/client.js";
|
|
2
|
+
import { mapDistanceToIndexParam } from "../utils/distance.js";
|
|
3
|
+
import { COLLECTION_STORAGE_MODE, isOneTableMode, } from "../config/env.js";
|
|
4
|
+
import { GLOBAL_POINTS_TABLE } from "../ydb/schema.js";
|
|
5
|
+
import { uidFor } from "../utils/tenant.js";
|
|
6
|
+
import { createCollectionMultiTable, deleteCollectionMultiTable, } from "./collectionsRepo.multi-table.js";
|
|
7
|
+
import { createCollectionOneTable, deleteCollectionOneTable, } from "./collectionsRepo.one-table.js";
|
|
8
|
+
export async function createCollection(metaKey, dim, distance, vectorType, tableName, layout = COLLECTION_STORAGE_MODE) {
|
|
9
|
+
if (isOneTableMode(layout)) {
|
|
10
|
+
await createCollectionOneTable(metaKey, dim, distance, vectorType);
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
await createCollectionMultiTable(metaKey, dim, distance, vectorType, tableName);
|
|
29
14
|
}
|
|
30
15
|
export async function getCollectionMeta(metaKey) {
|
|
31
16
|
const qry = `
|
|
@@ -49,20 +34,23 @@ export async function getCollectionMeta(metaKey) {
|
|
|
49
34
|
const vectorType = row.items?.[3]?.textValue ?? "float";
|
|
50
35
|
return { table, dimension, distance, vectorType };
|
|
51
36
|
}
|
|
52
|
-
export async function deleteCollection(metaKey) {
|
|
37
|
+
export async function deleteCollection(metaKey, uid) {
|
|
53
38
|
const meta = await getCollectionMeta(metaKey);
|
|
54
39
|
if (!meta)
|
|
55
40
|
return;
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
41
|
+
if (meta.table === GLOBAL_POINTS_TABLE) {
|
|
42
|
+
const effectiveUid = uid ??
|
|
43
|
+
(() => {
|
|
44
|
+
const [tenant, collection] = metaKey.split("/", 2);
|
|
45
|
+
if (!tenant || !collection) {
|
|
46
|
+
throw new Error(`deleteCollection: cannot derive uid from malformed metaKey=${metaKey}`);
|
|
47
|
+
}
|
|
48
|
+
return uidFor(tenant, collection);
|
|
49
|
+
})();
|
|
50
|
+
await deleteCollectionOneTable(metaKey, effectiveUid);
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
await deleteCollectionMultiTable(metaKey, meta.table);
|
|
66
54
|
}
|
|
67
55
|
export async function buildVectorIndex(tableName, dimension, distance, vectorType) {
|
|
68
56
|
const distParam = mapDistanceToIndexParam(distance);
|
|
@@ -103,17 +91,3 @@ export async function buildVectorIndex(tableName, dimension, distance, vectorTyp
|
|
|
103
91
|
await rawSession.api.executeSchemeQuery(createReq);
|
|
104
92
|
});
|
|
105
93
|
}
|
|
106
|
-
function mapDistanceToIndexParam(distance) {
|
|
107
|
-
switch (distance) {
|
|
108
|
-
case "Cosine":
|
|
109
|
-
return "cosine";
|
|
110
|
-
case "Dot":
|
|
111
|
-
return "inner_product";
|
|
112
|
-
case "Euclid":
|
|
113
|
-
return "euclidean";
|
|
114
|
-
case "Manhattan":
|
|
115
|
-
return "manhattan";
|
|
116
|
-
default:
|
|
117
|
-
return "cosine";
|
|
118
|
-
}
|
|
119
|
-
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { DistanceKind, VectorType } from "../types";
|
|
2
|
+
export declare function createCollectionMultiTable(metaKey: string, dim: number, distance: DistanceKind, vectorType: VectorType, tableName: string): Promise<void>;
|
|
3
|
+
export declare function deleteCollectionMultiTable(metaKey: string, tableName: string): Promise<void>;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { Types, TypedValues, withSession, TableDescription, Column, } from "../ydb/client.js";
|
|
2
|
+
import { upsertCollectionMeta } from "./collectionsRepo.shared.js";
|
|
3
|
+
export async function createCollectionMultiTable(metaKey, dim, distance, vectorType, tableName) {
|
|
4
|
+
await withSession(async (s) => {
|
|
5
|
+
const desc = new TableDescription()
|
|
6
|
+
.withColumns(new Column("point_id", Types.UTF8), new Column("embedding", Types.BYTES), new Column("payload", Types.JSON_DOCUMENT))
|
|
7
|
+
.withPrimaryKey("point_id");
|
|
8
|
+
await s.createTable(tableName, desc);
|
|
9
|
+
});
|
|
10
|
+
await upsertCollectionMeta(metaKey, dim, distance, vectorType, tableName);
|
|
11
|
+
}
|
|
12
|
+
export async function deleteCollectionMultiTable(metaKey, tableName) {
|
|
13
|
+
await withSession(async (s) => {
|
|
14
|
+
await s.dropTable(tableName);
|
|
15
|
+
});
|
|
16
|
+
const delMeta = `
|
|
17
|
+
DECLARE $collection AS Utf8;
|
|
18
|
+
DELETE FROM qdr__collections WHERE collection = $collection;
|
|
19
|
+
`;
|
|
20
|
+
await withSession(async (s) => {
|
|
21
|
+
await s.executeQuery(delMeta, { $collection: TypedValues.utf8(metaKey) });
|
|
22
|
+
});
|
|
23
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { DistanceKind, VectorType } from "../types";
|
|
2
|
+
export declare function createCollectionOneTable(metaKey: string, dim: number, distance: DistanceKind, vectorType: VectorType): Promise<void>;
|
|
3
|
+
export declare function deleteCollectionOneTable(metaKey: string, uid: string): Promise<void>;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { TypedValues, withSession } from "../ydb/client.js";
|
|
2
|
+
import { GLOBAL_POINTS_TABLE, ensureGlobalPointsTable } from "../ydb/schema.js";
|
|
3
|
+
import { upsertCollectionMeta } from "./collectionsRepo.shared.js";
|
|
4
|
+
export async function createCollectionOneTable(metaKey, dim, distance, vectorType) {
|
|
5
|
+
await upsertCollectionMeta(metaKey, dim, distance, vectorType, GLOBAL_POINTS_TABLE);
|
|
6
|
+
}
|
|
7
|
+
export async function deleteCollectionOneTable(metaKey, uid) {
|
|
8
|
+
await ensureGlobalPointsTable();
|
|
9
|
+
const deletePointsYql = `
|
|
10
|
+
DECLARE $uid AS Utf8;
|
|
11
|
+
DELETE FROM ${GLOBAL_POINTS_TABLE} WHERE uid = $uid;
|
|
12
|
+
`;
|
|
13
|
+
await withSession(async (s) => {
|
|
14
|
+
await s.executeQuery(deletePointsYql, {
|
|
15
|
+
$uid: TypedValues.utf8(uid),
|
|
16
|
+
});
|
|
17
|
+
});
|
|
18
|
+
const delMeta = `
|
|
19
|
+
DECLARE $collection AS Utf8;
|
|
20
|
+
DELETE FROM qdr__collections WHERE collection = $collection;
|
|
21
|
+
`;
|
|
22
|
+
await withSession(async (s) => {
|
|
23
|
+
await s.executeQuery(delMeta, { $collection: TypedValues.utf8(metaKey) });
|
|
24
|
+
});
|
|
25
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { TypedValues, withSession } from "../ydb/client.js";
|
|
2
|
+
export async function upsertCollectionMeta(metaKey, dim, distance, vectorType, tableName) {
|
|
3
|
+
const upsertMeta = `
|
|
4
|
+
DECLARE $collection AS Utf8;
|
|
5
|
+
DECLARE $table AS Utf8;
|
|
6
|
+
DECLARE $dim AS Uint32;
|
|
7
|
+
DECLARE $distance AS Utf8;
|
|
8
|
+
DECLARE $vtype AS Utf8;
|
|
9
|
+
DECLARE $created AS Timestamp;
|
|
10
|
+
UPSERT INTO qdr__collections (collection, table_name, vector_dimension, distance, vector_type, created_at)
|
|
11
|
+
VALUES ($collection, $table, $dim, $distance, $vtype, $created);
|
|
12
|
+
`;
|
|
13
|
+
await withSession(async (s) => {
|
|
14
|
+
await s.executeQuery(upsertMeta, {
|
|
15
|
+
$collection: TypedValues.utf8(metaKey),
|
|
16
|
+
$table: TypedValues.utf8(tableName),
|
|
17
|
+
$dim: TypedValues.uint32(dim),
|
|
18
|
+
$distance: TypedValues.utf8(distance),
|
|
19
|
+
$vtype: TypedValues.utf8(vectorType),
|
|
20
|
+
$created: TypedValues.timestamp(new Date()),
|
|
21
|
+
});
|
|
22
|
+
});
|
|
23
|
+
}
|
|
@@ -3,10 +3,10 @@ export declare function upsertPoints(tableName: string, points: Array<{
|
|
|
3
3
|
id: string | number;
|
|
4
4
|
vector: number[];
|
|
5
5
|
payload?: Record<string, unknown>;
|
|
6
|
-
}>, dimension: number): Promise<number>;
|
|
7
|
-
export declare function searchPoints(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number): Promise<Array<{
|
|
6
|
+
}>, dimension: number, uid?: string): Promise<number>;
|
|
7
|
+
export declare function searchPoints(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number, uid?: string): Promise<Array<{
|
|
8
8
|
id: string;
|
|
9
9
|
score: number;
|
|
10
10
|
payload?: Record<string, unknown>;
|
|
11
11
|
}>>;
|
|
12
|
-
export declare function deletePoints(tableName: string, ids: Array<string | number
|
|
12
|
+
export declare function deletePoints(tableName: string, ids: Array<string | number>, uid?: string): Promise<number>;
|