ydb-qdrant 6.0.0 → 8.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/env.d.ts +0 -3
- package/dist/config/env.js +0 -17
- package/dist/package/api.d.ts +3 -0
- package/dist/qdrant/QdrantRestTypes.d.ts +35 -0
- package/dist/qdrant/QdrantRestTypes.js +1 -0
- package/dist/repositories/collectionsRepo.one-table.js +37 -63
- package/dist/repositories/collectionsRepo.shared.js +8 -2
- package/dist/repositories/pointsRepo.d.ts +5 -11
- package/dist/repositories/pointsRepo.js +6 -3
- package/dist/repositories/pointsRepo.one-table/Delete.d.ts +2 -0
- package/dist/repositories/pointsRepo.one-table/Delete.js +166 -0
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.d.ts +14 -0
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.js +33 -0
- package/dist/repositories/pointsRepo.one-table/Search.d.ts +4 -0
- package/dist/repositories/pointsRepo.one-table/Search.js +208 -0
- package/dist/repositories/pointsRepo.one-table/Upsert.d.ts +2 -0
- package/dist/repositories/pointsRepo.one-table/Upsert.js +85 -0
- package/dist/repositories/pointsRepo.one-table.d.ts +3 -13
- package/dist/repositories/pointsRepo.one-table.js +3 -403
- package/dist/routes/points.js +17 -4
- package/dist/server.d.ts +1 -0
- package/dist/server.js +70 -2
- package/dist/services/CollectionService.d.ts +9 -0
- package/dist/services/CollectionService.js +9 -0
- package/dist/services/PointsService.d.ts +3 -10
- package/dist/services/PointsService.js +73 -3
- package/dist/types.d.ts +59 -5
- package/dist/types.js +27 -3
- package/dist/utils/normalization.d.ts +1 -0
- package/dist/utils/normalization.js +2 -1
- package/dist/utils/vectorBinary.js +94 -10
- package/dist/ydb/bootstrapMetaTable.d.ts +7 -0
- package/dist/ydb/bootstrapMetaTable.js +75 -0
- package/dist/ydb/client.d.ts +10 -3
- package/dist/ydb/client.js +26 -2
- package/dist/ydb/helpers.d.ts +0 -2
- package/dist/ydb/helpers.js +0 -7
- package/dist/ydb/schema.js +100 -66
- package/package.json +3 -6
package/dist/config/env.d.ts
CHANGED
|
@@ -3,8 +3,6 @@ export declare const YDB_ENDPOINT: string;
|
|
|
3
3
|
export declare const YDB_DATABASE: string;
|
|
4
4
|
export declare const PORT: number;
|
|
5
5
|
export declare const LOG_LEVEL: string;
|
|
6
|
-
export declare const GLOBAL_POINTS_AUTOMIGRATE_ENABLED: boolean;
|
|
7
|
-
export declare const USE_BATCH_DELETE_FOR_COLLECTIONS: boolean;
|
|
8
6
|
export declare enum SearchMode {
|
|
9
7
|
Exact = "exact",
|
|
10
8
|
Approximate = "approximate"
|
|
@@ -12,7 +10,6 @@ export declare enum SearchMode {
|
|
|
12
10
|
export declare function resolveSearchMode(raw: string | undefined): SearchMode;
|
|
13
11
|
export declare const SEARCH_MODE: SearchMode;
|
|
14
12
|
export declare const OVERFETCH_MULTIPLIER: number;
|
|
15
|
-
export declare const CLIENT_SIDE_SERIALIZATION_ENABLED: boolean;
|
|
16
13
|
export declare const UPSERT_BATCH_SIZE: number;
|
|
17
14
|
export declare const SESSION_POOL_MIN_SIZE: number;
|
|
18
15
|
export declare const SESSION_POOL_MAX_SIZE: number;
|
package/dist/config/env.js
CHANGED
|
@@ -20,22 +20,6 @@ function parseIntegerEnv(value, defaultValue, opts) {
|
|
|
20
20
|
}
|
|
21
21
|
return result;
|
|
22
22
|
}
|
|
23
|
-
function parseBooleanEnv(value, defaultValue) {
|
|
24
|
-
if (value === undefined) {
|
|
25
|
-
return defaultValue;
|
|
26
|
-
}
|
|
27
|
-
const normalized = value.trim().toLowerCase();
|
|
28
|
-
if (normalized === "" ||
|
|
29
|
-
normalized === "0" ||
|
|
30
|
-
normalized === "false" ||
|
|
31
|
-
normalized === "no" ||
|
|
32
|
-
normalized === "off") {
|
|
33
|
-
return false;
|
|
34
|
-
}
|
|
35
|
-
return true;
|
|
36
|
-
}
|
|
37
|
-
export const GLOBAL_POINTS_AUTOMIGRATE_ENABLED = parseBooleanEnv(process.env.YDB_QDRANT_GLOBAL_POINTS_AUTOMIGRATE, false);
|
|
38
|
-
export const USE_BATCH_DELETE_FOR_COLLECTIONS = parseBooleanEnv(process.env.YDB_QDRANT_USE_BATCH_DELETE, false);
|
|
39
23
|
export var SearchMode;
|
|
40
24
|
(function (SearchMode) {
|
|
41
25
|
SearchMode["Exact"] = "exact";
|
|
@@ -57,7 +41,6 @@ function resolveSearchModeEnv() {
|
|
|
57
41
|
}
|
|
58
42
|
export const SEARCH_MODE = resolveSearchModeEnv();
|
|
59
43
|
export const OVERFETCH_MULTIPLIER = parseIntegerEnv(process.env.YDB_QDRANT_OVERFETCH_MULTIPLIER, 10, { min: 1 });
|
|
60
|
-
export const CLIENT_SIDE_SERIALIZATION_ENABLED = parseBooleanEnv(process.env.YDB_QDRANT_CLIENT_SIDE_SERIALIZATION_ENABLED, false);
|
|
61
44
|
export const UPSERT_BATCH_SIZE = parseIntegerEnv(process.env.YDB_QDRANT_UPSERT_BATCH_SIZE, 100, { min: 1 });
|
|
62
45
|
// Session pool configuration
|
|
63
46
|
const RAW_SESSION_POOL_MIN_SIZE = parseIntegerEnv(process.env.YDB_SESSION_POOL_MIN_SIZE, 5, { min: 1, max: 500 });
|
package/dist/package/api.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { createCollection as serviceCreateCollection, deleteCollection as servic
|
|
|
3
3
|
import { upsertPoints as serviceUpsertPoints, searchPoints as serviceSearchPoints, deletePoints as serviceDeletePoints } from "../services/PointsService.js";
|
|
4
4
|
export { QdrantServiceError } from "../services/errors.js";
|
|
5
5
|
export { CreateCollectionReq, UpsertPointsReq, SearchReq, DeletePointsReq, } from "../types.js";
|
|
6
|
+
export type { UpsertPointsBody, SearchPointsBody } from "../types.js";
|
|
6
7
|
type CreateCollectionResult = Awaited<ReturnType<typeof serviceCreateCollection>>;
|
|
7
8
|
type GetCollectionResult = Awaited<ReturnType<typeof serviceGetCollection>>;
|
|
8
9
|
type DeleteCollectionResult = Awaited<ReturnType<typeof serviceDeleteCollection>>;
|
|
@@ -22,7 +23,9 @@ export interface YdbQdrantTenantClient {
|
|
|
22
23
|
getCollection(collection: string): Promise<GetCollectionResult>;
|
|
23
24
|
deleteCollection(collection: string): Promise<DeleteCollectionResult>;
|
|
24
25
|
putCollectionIndex(collection: string): Promise<PutIndexResult>;
|
|
26
|
+
upsertPoints(collection: string, body: import("../types.js").UpsertPointsBody): Promise<UpsertPointsResult>;
|
|
25
27
|
upsertPoints(collection: string, body: unknown): Promise<UpsertPointsResult>;
|
|
28
|
+
searchPoints(collection: string, body: import("../types.js").SearchPointsBody): Promise<SearchPointsResult>;
|
|
26
29
|
searchPoints(collection: string, body: unknown): Promise<SearchPointsResult>;
|
|
27
30
|
deletePoints(collection: string, body: unknown): Promise<DeletePointsResult>;
|
|
28
31
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { Schemas } from "@qdrant/js-client-rest";
|
|
2
|
+
/**
|
|
3
|
+
* Type-only surface derived from Qdrant's official REST OpenAPI schema.
|
|
4
|
+
*
|
|
5
|
+
* Important:
|
|
6
|
+
* - This module must remain type-only (no runtime imports/exports from @qdrant/*).
|
|
7
|
+
* - Prefer referencing these aliases across the codebase to keep our shapes aligned
|
|
8
|
+
* with upstream Qdrant where appropriate, while preserving our runtime behavior.
|
|
9
|
+
*/
|
|
10
|
+
export type QdrantSchemas = Schemas;
|
|
11
|
+
export type QdrantDistance = Schemas["Distance"];
|
|
12
|
+
export type QdrantExtendedPointId = Schemas["ExtendedPointId"];
|
|
13
|
+
export type QdrantPayload = Schemas["Payload"];
|
|
14
|
+
export type QdrantFilter = Schemas["Filter"];
|
|
15
|
+
export type QdrantPointsSelector = Schemas["PointsSelector"];
|
|
16
|
+
export type QdrantPointStruct = Schemas["PointStruct"];
|
|
17
|
+
export type QdrantSearchRequest = Schemas["SearchRequest"];
|
|
18
|
+
export type QdrantScoredPoint = Schemas["ScoredPoint"];
|
|
19
|
+
export type QdrantWithPayloadInterface = Schemas["WithPayloadInterface"];
|
|
20
|
+
/**
|
|
21
|
+
* Project-specific narrowing for our Qdrant-compatible subset.
|
|
22
|
+
* These types are intentionally tighter than Qdrant's full schema.
|
|
23
|
+
*/
|
|
24
|
+
export type YdbQdrantPointId = Extract<QdrantExtendedPointId, string | number>;
|
|
25
|
+
export type YdbQdrantVector = number[];
|
|
26
|
+
export type YdbQdrantUpsertPoint = Omit<QdrantPointStruct, "id" | "vector" | "payload"> & {
|
|
27
|
+
id: YdbQdrantPointId;
|
|
28
|
+
vector: YdbQdrantVector;
|
|
29
|
+
payload?: QdrantPayload;
|
|
30
|
+
};
|
|
31
|
+
export type YdbQdrantScoredPoint = {
|
|
32
|
+
id: string;
|
|
33
|
+
score: QdrantScoredPoint["score"];
|
|
34
|
+
payload?: QdrantPayload;
|
|
35
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import { TypedValues, Types, withSession, createExecuteQuerySettings, } from "../ydb/client.js";
|
|
1
|
+
import { TypedValues, Types, withSession, withQuerySession, createExecuteQuerySettings, } from "../ydb/client.js";
|
|
2
2
|
import { GLOBAL_POINTS_TABLE, ensureGlobalPointsTable } from "../ydb/schema.js";
|
|
3
3
|
import { upsertCollectionMeta } from "./collectionsRepo.shared.js";
|
|
4
4
|
import { withRetry, isTransientYdbError } from "../utils/retry.js";
|
|
5
|
-
import { USE_BATCH_DELETE_FOR_COLLECTIONS } from "../config/env.js";
|
|
6
5
|
const DELETE_COLLECTION_BATCH_SIZE = 10000;
|
|
7
6
|
function isOutOfBufferMemoryYdbError(error) {
|
|
8
7
|
const msg = error instanceof Error ? error.message : String(error);
|
|
@@ -37,7 +36,7 @@ async function deletePointsForUidInChunks(s, uid) {
|
|
|
37
36
|
// Each iteration only touches a limited number of rows to avoid
|
|
38
37
|
// hitting YDB's per‑operation buffer limits.
|
|
39
38
|
let iterations = 0;
|
|
40
|
-
const MAX_ITERATIONS =
|
|
39
|
+
const MAX_ITERATIONS = 10000;
|
|
41
40
|
const settings = createExecuteQuerySettings();
|
|
42
41
|
while (iterations++ < MAX_ITERATIONS) {
|
|
43
42
|
const rs = (await s.executeQuery(selectYql, {
|
|
@@ -64,68 +63,43 @@ export async function createCollectionOneTable(metaKey, dim, distance, vectorTyp
|
|
|
64
63
|
}
|
|
65
64
|
export async function deleteCollectionOneTable(metaKey, uid) {
|
|
66
65
|
await ensureGlobalPointsTable();
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
// the same per-uid chunked deletion strategy as the legacy path.
|
|
87
|
-
await deletePointsForUidInChunks(s, uid);
|
|
88
|
-
}
|
|
89
|
-
}), {
|
|
90
|
-
isTransient: isTransientYdbError,
|
|
91
|
-
context: {
|
|
92
|
-
operation: "deleteCollectionOneTable",
|
|
93
|
-
tableName: GLOBAL_POINTS_TABLE,
|
|
94
|
-
metaKey,
|
|
95
|
-
uid,
|
|
96
|
-
mode: "batch_delete",
|
|
97
|
-
},
|
|
98
|
-
});
|
|
99
|
-
}
|
|
100
|
-
else {
|
|
101
|
-
const deletePointsYql = `
|
|
102
|
-
DECLARE $uid AS Utf8;
|
|
103
|
-
DELETE FROM ${GLOBAL_POINTS_TABLE} WHERE uid = $uid;
|
|
104
|
-
`;
|
|
105
|
-
await withRetry(() => withSession(async (s) => {
|
|
106
|
-
const settings = createExecuteQuerySettings();
|
|
107
|
-
try {
|
|
108
|
-
await s.executeQuery(deletePointsYql, {
|
|
109
|
-
$uid: TypedValues.utf8(uid),
|
|
110
|
-
}, undefined, settings);
|
|
66
|
+
const batchDeletePointsYql = `
|
|
67
|
+
DECLARE $uid AS Utf8;
|
|
68
|
+
BATCH DELETE FROM ${GLOBAL_POINTS_TABLE}
|
|
69
|
+
WHERE uid = $uid;
|
|
70
|
+
`;
|
|
71
|
+
await withRetry(async () => {
|
|
72
|
+
try {
|
|
73
|
+
await withQuerySession(async (qs) => {
|
|
74
|
+
await qs.execute({
|
|
75
|
+
text: batchDeletePointsYql,
|
|
76
|
+
parameters: {
|
|
77
|
+
$uid: TypedValues.utf8(uid),
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
catch (err) {
|
|
83
|
+
if (!isOutOfBufferMemoryYdbError(err)) {
|
|
84
|
+
throw err;
|
|
111
85
|
}
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
86
|
+
// BATCH DELETE already deletes in chunks per partition, but if YDB
|
|
87
|
+
// still reports an out-of-buffer-memory condition, fall back to
|
|
88
|
+
// per-uid chunked deletion strategy to complete the deletion.
|
|
89
|
+
await withSession(async (s) => {
|
|
116
90
|
await deletePointsForUidInChunks(s, uid);
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
}
|
|
128
|
-
}
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
}, {
|
|
94
|
+
isTransient: isTransientYdbError,
|
|
95
|
+
context: {
|
|
96
|
+
operation: "deleteCollectionOneTable",
|
|
97
|
+
tableName: GLOBAL_POINTS_TABLE,
|
|
98
|
+
metaKey,
|
|
99
|
+
uid,
|
|
100
|
+
mode: "batch_delete",
|
|
101
|
+
},
|
|
102
|
+
});
|
|
129
103
|
const delMeta = `
|
|
130
104
|
DECLARE $collection AS Utf8;
|
|
131
105
|
DELETE FROM qdr__collections WHERE collection = $collection;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { UPSERT_OPERATION_TIMEOUT_MS } from "../config/env.js";
|
|
2
|
+
import { TypedValues, withSession, createExecuteQuerySettingsWithTimeout, } from "../ydb/client.js";
|
|
2
3
|
export async function upsertCollectionMeta(metaKey, dim, distance, vectorType, tableName) {
|
|
3
4
|
const now = new Date();
|
|
4
5
|
const upsertMeta = `
|
|
@@ -13,6 +14,11 @@ export async function upsertCollectionMeta(metaKey, dim, distance, vectorType, t
|
|
|
13
14
|
VALUES ($collection, $table, $dim, $distance, $vtype, $created, $last_accessed);
|
|
14
15
|
`;
|
|
15
16
|
await withSession(async (s) => {
|
|
17
|
+
const settings = createExecuteQuerySettingsWithTimeout({
|
|
18
|
+
keepInCache: true,
|
|
19
|
+
idempotent: true,
|
|
20
|
+
timeoutMs: UPSERT_OPERATION_TIMEOUT_MS,
|
|
21
|
+
});
|
|
16
22
|
await s.executeQuery(upsertMeta, {
|
|
17
23
|
$collection: TypedValues.utf8(metaKey),
|
|
18
24
|
$table: TypedValues.utf8(tableName),
|
|
@@ -21,6 +27,6 @@ export async function upsertCollectionMeta(metaKey, dim, distance, vectorType, t
|
|
|
21
27
|
$vtype: TypedValues.utf8(vectorType),
|
|
22
28
|
$created: TypedValues.timestamp(now),
|
|
23
29
|
$last_accessed: TypedValues.timestamp(now),
|
|
24
|
-
});
|
|
30
|
+
}, undefined, settings);
|
|
25
31
|
});
|
|
26
32
|
}
|
|
@@ -1,12 +1,6 @@
|
|
|
1
|
-
import type { DistanceKind } from "../types";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
payload?: Record<string, unknown>;
|
|
6
|
-
}>, dimension: number, uid: string): Promise<number>;
|
|
7
|
-
export declare function searchPoints(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number, uid: string): Promise<Array<{
|
|
8
|
-
id: string;
|
|
9
|
-
score: number;
|
|
10
|
-
payload?: Record<string, unknown>;
|
|
11
|
-
}>>;
|
|
1
|
+
import type { DistanceKind, UpsertPoint } from "../types.js";
|
|
2
|
+
import type { YdbQdrantScoredPoint } from "../qdrant/QdrantRestTypes.js";
|
|
3
|
+
export declare function upsertPoints(tableName: string, points: UpsertPoint[], dimension: number, uid: string): Promise<number>;
|
|
4
|
+
export declare function searchPoints(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number, uid: string, filterPaths?: Array<Array<string>>): Promise<YdbQdrantScoredPoint[]>;
|
|
12
5
|
export declare function deletePoints(tableName: string, ids: Array<string | number>, uid: string): Promise<number>;
|
|
6
|
+
export declare function deletePointsByPathSegments(tableName: string, uid: string, paths: Array<Array<string>>): Promise<number>;
|
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
import { SEARCH_MODE, OVERFETCH_MULTIPLIER, } from "../config/env.js";
|
|
2
|
-
import { upsertPointsOneTable, searchPointsOneTable, deletePointsOneTable, } from "./pointsRepo.one-table.js";
|
|
2
|
+
import { upsertPointsOneTable, searchPointsOneTable, deletePointsOneTable, deletePointsByPathSegmentsOneTable, } from "./pointsRepo.one-table.js";
|
|
3
3
|
export async function upsertPoints(tableName, points, dimension, uid) {
|
|
4
4
|
return await upsertPointsOneTable(tableName, points, dimension, uid);
|
|
5
5
|
}
|
|
6
|
-
export async function searchPoints(tableName, queryVector, top, withPayload, distance, dimension, uid) {
|
|
6
|
+
export async function searchPoints(tableName, queryVector, top, withPayload, distance, dimension, uid, filterPaths) {
|
|
7
7
|
const mode = SEARCH_MODE;
|
|
8
|
-
return await searchPointsOneTable(tableName, queryVector, top, withPayload, distance, dimension, uid, mode, OVERFETCH_MULTIPLIER);
|
|
8
|
+
return await searchPointsOneTable(tableName, queryVector, top, withPayload, distance, dimension, uid, mode, OVERFETCH_MULTIPLIER, filterPaths);
|
|
9
9
|
}
|
|
10
10
|
export async function deletePoints(tableName, ids, uid) {
|
|
11
11
|
return await deletePointsOneTable(tableName, ids, uid);
|
|
12
12
|
}
|
|
13
|
+
export async function deletePointsByPathSegments(tableName, uid, paths) {
|
|
14
|
+
return await deletePointsByPathSegmentsOneTable(tableName, uid, paths);
|
|
15
|
+
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import { TypedValues, withSession, createExecuteQuerySettings, } from "../../ydb/client.js";
|
|
2
|
+
import { withRetry, isTransientYdbError } from "../../utils/retry.js";
|
|
3
|
+
import { buildPathSegmentsWhereClause } from "./PathSegmentsFilter.js";
|
|
4
|
+
const DELETE_FILTER_SELECT_BATCH_SIZE = 1000;
|
|
5
|
+
export async function deletePointsOneTable(tableName, ids, uid) {
|
|
6
|
+
let deleted = 0;
|
|
7
|
+
await withSession(async (s) => {
|
|
8
|
+
const settings = createExecuteQuerySettings();
|
|
9
|
+
for (const id of ids) {
|
|
10
|
+
const yql = `
|
|
11
|
+
DECLARE $uid AS Utf8;
|
|
12
|
+
DECLARE $id AS Utf8;
|
|
13
|
+
DELETE FROM ${tableName} WHERE uid = $uid AND point_id = $id;
|
|
14
|
+
`;
|
|
15
|
+
const params = {
|
|
16
|
+
$uid: TypedValues.utf8(uid),
|
|
17
|
+
$id: TypedValues.utf8(String(id)),
|
|
18
|
+
};
|
|
19
|
+
await withRetry(() => s.executeQuery(yql, params, undefined, settings), {
|
|
20
|
+
isTransient: isTransientYdbError,
|
|
21
|
+
context: { tableName, uid, pointId: String(id) },
|
|
22
|
+
});
|
|
23
|
+
deleted += 1;
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
return deleted;
|
|
27
|
+
}
|
|
28
|
+
const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
|
|
29
|
+
function bigintToSafeNumberOrNull(value) {
|
|
30
|
+
if (value > MAX_SAFE_BIGINT || value < -MAX_SAFE_BIGINT) {
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
return Number(value);
|
|
34
|
+
}
|
|
35
|
+
function longLikeToBigInt(value) {
|
|
36
|
+
const low = BigInt(value.low >>> 0);
|
|
37
|
+
const high = BigInt(value.high >>> 0);
|
|
38
|
+
let n = low + (high << 32n);
|
|
39
|
+
// If this is a signed Long-like and the sign bit is set, interpret as a negative 64-bit integer.
|
|
40
|
+
const isUnsigned = value.unsigned === true;
|
|
41
|
+
const signBitSet = (value.high & 0x8000_0000) !== 0;
|
|
42
|
+
if (!isUnsigned && signBitSet) {
|
|
43
|
+
n -= 1n << 64n;
|
|
44
|
+
}
|
|
45
|
+
return n;
|
|
46
|
+
}
|
|
47
|
+
function toNumber(value) {
|
|
48
|
+
if (typeof value === "number" && Number.isFinite(value))
|
|
49
|
+
return value;
|
|
50
|
+
if (typeof value === "bigint") {
|
|
51
|
+
return bigintToSafeNumberOrNull(value);
|
|
52
|
+
}
|
|
53
|
+
if (typeof value === "string") {
|
|
54
|
+
// Prefer exact parsing for integer strings to avoid silent precision loss.
|
|
55
|
+
if (/^-?\d+$/.test(value.trim())) {
|
|
56
|
+
try {
|
|
57
|
+
const b = BigInt(value.trim());
|
|
58
|
+
return bigintToSafeNumberOrNull(b);
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
const n = Number(value);
|
|
65
|
+
return Number.isFinite(n) ? n : null;
|
|
66
|
+
}
|
|
67
|
+
if (value && typeof value === "object") {
|
|
68
|
+
// ydb-sdk may return Uint64/Int64 as protobufjs Long-like objects:
|
|
69
|
+
// { low: number, high: number, unsigned?: boolean }
|
|
70
|
+
const v = value;
|
|
71
|
+
if (typeof v.low === "number" && typeof v.high === "number") {
|
|
72
|
+
const b = longLikeToBigInt({
|
|
73
|
+
low: v.low,
|
|
74
|
+
high: v.high,
|
|
75
|
+
unsigned: v.unsigned === true,
|
|
76
|
+
});
|
|
77
|
+
return bigintToSafeNumberOrNull(b);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
function readDeletedCountFromResult(rs) {
|
|
83
|
+
const sets = rs.resultSets ?? [];
|
|
84
|
+
for (let i = sets.length - 1; i >= 0; i -= 1) {
|
|
85
|
+
const rowset = sets[i];
|
|
86
|
+
const rows = rowset?.rows ?? [];
|
|
87
|
+
const cell = rows[0]?.items?.[0];
|
|
88
|
+
if (!cell)
|
|
89
|
+
continue;
|
|
90
|
+
const candidates = [
|
|
91
|
+
cell.uint64Value,
|
|
92
|
+
cell.int64Value,
|
|
93
|
+
cell.uint32Value,
|
|
94
|
+
cell.int32Value,
|
|
95
|
+
cell.textValue,
|
|
96
|
+
];
|
|
97
|
+
for (const c of candidates) {
|
|
98
|
+
const n = toNumber(c);
|
|
99
|
+
if (n !== null)
|
|
100
|
+
return n;
|
|
101
|
+
}
|
|
102
|
+
// We got a result cell but couldn't parse any of its known numeric representations.
|
|
103
|
+
// Returning 0 here would silently stop the delete loop, so fail loud.
|
|
104
|
+
throw new Error("Unable to parse deleted count from YDB result.");
|
|
105
|
+
}
|
|
106
|
+
return 0;
|
|
107
|
+
}
|
|
108
|
+
export async function deletePointsByPathSegmentsOneTable(tableName, uid, paths) {
|
|
109
|
+
if (paths.length === 0) {
|
|
110
|
+
return 0;
|
|
111
|
+
}
|
|
112
|
+
const { whereSql, params: whereParams } = buildPathSegmentsWhereClause(paths);
|
|
113
|
+
const whereParamDeclarations = Object.keys(whereParams)
|
|
114
|
+
.sort()
|
|
115
|
+
.map((key) => `DECLARE ${key} AS Utf8;`)
|
|
116
|
+
.join("\n ");
|
|
117
|
+
const deleteBatchYql = `
|
|
118
|
+
DECLARE $uid AS Utf8;
|
|
119
|
+
DECLARE $limit AS Uint32;
|
|
120
|
+
${whereParamDeclarations}
|
|
121
|
+
|
|
122
|
+
$to_delete = (
|
|
123
|
+
SELECT uid, point_id
|
|
124
|
+
FROM ${tableName}
|
|
125
|
+
WHERE uid = $uid AND ${whereSql}
|
|
126
|
+
LIMIT $limit
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
DELETE FROM ${tableName} ON
|
|
130
|
+
SELECT uid, point_id FROM $to_delete;
|
|
131
|
+
|
|
132
|
+
SELECT CAST(COUNT(*) AS Uint32) AS deleted FROM $to_delete;
|
|
133
|
+
`;
|
|
134
|
+
let deleted = 0;
|
|
135
|
+
await withSession(async (s) => {
|
|
136
|
+
const settings = createExecuteQuerySettings();
|
|
137
|
+
// Best-effort loop: stop when there are no more matching rows.
|
|
138
|
+
// Use limited batches to avoid per-operation buffer limits.
|
|
139
|
+
while (true) {
|
|
140
|
+
const rs = (await withRetry(() => s.executeQuery(deleteBatchYql, {
|
|
141
|
+
...whereParams,
|
|
142
|
+
$uid: TypedValues.utf8(uid),
|
|
143
|
+
$limit: TypedValues.uint32(DELETE_FILTER_SELECT_BATCH_SIZE),
|
|
144
|
+
}, undefined, settings), {
|
|
145
|
+
isTransient: isTransientYdbError,
|
|
146
|
+
context: {
|
|
147
|
+
tableName,
|
|
148
|
+
uid,
|
|
149
|
+
filterPathsCount: paths.length,
|
|
150
|
+
batchLimit: DELETE_FILTER_SELECT_BATCH_SIZE,
|
|
151
|
+
},
|
|
152
|
+
}));
|
|
153
|
+
const batchDeleted = readDeletedCountFromResult(rs);
|
|
154
|
+
if (!Number.isSafeInteger(batchDeleted) ||
|
|
155
|
+
batchDeleted < 0 ||
|
|
156
|
+
batchDeleted > DELETE_FILTER_SELECT_BATCH_SIZE) {
|
|
157
|
+
throw new Error(`Unexpected deleted count from YDB: ${String(batchDeleted)}. Expected an integer in [0, ${DELETE_FILTER_SELECT_BATCH_SIZE}].`);
|
|
158
|
+
}
|
|
159
|
+
if (batchDeleted <= 0) {
|
|
160
|
+
break;
|
|
161
|
+
}
|
|
162
|
+
deleted += batchDeleted;
|
|
163
|
+
}
|
|
164
|
+
});
|
|
165
|
+
return deleted;
|
|
166
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { Ydb } from "ydb-sdk";
|
|
2
|
+
type QueryParams = {
|
|
3
|
+
[key: string]: Ydb.ITypedValue;
|
|
4
|
+
};
|
|
5
|
+
export declare function buildPathSegmentsWhereClause(paths: Array<Array<string>>): {
|
|
6
|
+
whereSql: string;
|
|
7
|
+
params: QueryParams;
|
|
8
|
+
};
|
|
9
|
+
export declare function buildPathSegmentsFilter(paths: Array<Array<string>> | undefined): {
|
|
10
|
+
whereSql: string;
|
|
11
|
+
whereParamDeclarations: string;
|
|
12
|
+
whereParams: QueryParams;
|
|
13
|
+
} | undefined;
|
|
14
|
+
export {};
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { TypedValues } from "../../ydb/client.js";
|
|
2
|
+
export function buildPathSegmentsWhereClause(paths) {
|
|
3
|
+
const params = {};
|
|
4
|
+
const orGroups = [];
|
|
5
|
+
for (let pIdx = 0; pIdx < paths.length; pIdx += 1) {
|
|
6
|
+
const segs = paths[pIdx];
|
|
7
|
+
if (segs.length === 0) {
|
|
8
|
+
throw new Error("delete-by-filter: empty path segments");
|
|
9
|
+
}
|
|
10
|
+
const andParts = [];
|
|
11
|
+
for (let sIdx = 0; sIdx < segs.length; sIdx += 1) {
|
|
12
|
+
const paramName = `$p${pIdx}_${sIdx}`;
|
|
13
|
+
// payload is JsonDocument; JSON_VALUE supports JsonPath access.
|
|
14
|
+
andParts.push(`JSON_VALUE(payload, '$.pathSegments."${sIdx}"') = ${paramName}`);
|
|
15
|
+
params[paramName] = TypedValues.utf8(segs[sIdx]);
|
|
16
|
+
}
|
|
17
|
+
orGroups.push(`(${andParts.join(" AND ")})`);
|
|
18
|
+
}
|
|
19
|
+
return {
|
|
20
|
+
whereSql: orGroups.length === 1 ? orGroups[0] : `(${orGroups.join(" OR ")})`,
|
|
21
|
+
params,
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
export function buildPathSegmentsFilter(paths) {
|
|
25
|
+
if (!paths || paths.length === 0)
|
|
26
|
+
return undefined;
|
|
27
|
+
const { whereSql, params: whereParams } = buildPathSegmentsWhereClause(paths);
|
|
28
|
+
const whereParamDeclarations = Object.keys(whereParams)
|
|
29
|
+
.sort()
|
|
30
|
+
.map((key) => `DECLARE ${key} AS Utf8;`)
|
|
31
|
+
.join("\n ");
|
|
32
|
+
return { whereSql, whereParamDeclarations, whereParams };
|
|
33
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { DistanceKind } from "../../types.js";
|
|
2
|
+
import { SearchMode } from "../../config/env.js";
|
|
3
|
+
import type { YdbQdrantScoredPoint } from "../../qdrant/QdrantRestTypes.js";
|
|
4
|
+
export declare function searchPointsOneTable(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number, uid: string, mode: SearchMode | undefined, overfetchMultiplier: number, filterPaths?: Array<Array<string>>): Promise<YdbQdrantScoredPoint[]>;
|