ydb-qdrant 7.0.1 → 8.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/config/env.d.ts +0 -8
- package/dist/config/env.js +2 -29
- package/dist/package/api.d.ts +5 -2
- package/dist/package/api.js +2 -2
- package/dist/qdrant/QdrantRestTypes.d.ts +35 -0
- package/dist/repositories/collectionsRepo.d.ts +1 -2
- package/dist/repositories/collectionsRepo.js +62 -103
- package/dist/repositories/collectionsRepo.one-table.js +103 -47
- package/dist/repositories/collectionsRepo.shared.d.ts +2 -0
- package/dist/repositories/collectionsRepo.shared.js +32 -0
- package/dist/repositories/pointsRepo.d.ts +4 -8
- package/dist/repositories/pointsRepo.one-table/Delete.js +122 -67
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.d.ts +5 -2
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.js +7 -6
- package/dist/repositories/pointsRepo.one-table/Search.d.ts +4 -0
- package/dist/repositories/pointsRepo.one-table/Search.js +208 -0
- package/dist/repositories/pointsRepo.one-table/Upsert.d.ts +2 -2
- package/dist/repositories/pointsRepo.one-table/Upsert.js +51 -66
- package/dist/repositories/pointsRepo.one-table.d.ts +1 -1
- package/dist/repositories/pointsRepo.one-table.js +1 -1
- package/dist/routes/collections.js +7 -61
- package/dist/routes/points.js +11 -66
- package/dist/services/PointsService.d.ts +3 -8
- package/dist/services/PointsService.js +19 -23
- package/dist/types.d.ts +23 -33
- package/dist/types.js +18 -20
- package/dist/utils/normalization.js +13 -14
- package/dist/utils/retry.js +19 -29
- package/dist/utils/vectorBinary.js +10 -5
- package/dist/ydb/bootstrapMetaTable.d.ts +7 -0
- package/dist/ydb/bootstrapMetaTable.js +75 -0
- package/dist/ydb/client.d.ts +23 -17
- package/dist/ydb/client.js +82 -423
- package/dist/ydb/schema.js +88 -148
- package/package.json +2 -10
- package/dist/qdrant/QdrantTypes.d.ts +0 -19
- package/dist/repositories/pointsRepo.one-table/Search/Approximate.d.ts +0 -18
- package/dist/repositories/pointsRepo.one-table/Search/Approximate.js +0 -119
- package/dist/repositories/pointsRepo.one-table/Search/Exact.d.ts +0 -17
- package/dist/repositories/pointsRepo.one-table/Search/Exact.js +0 -101
- package/dist/repositories/pointsRepo.one-table/Search/index.d.ts +0 -8
- package/dist/repositories/pointsRepo.one-table/Search/index.js +0 -30
- package/dist/utils/typeGuards.d.ts +0 -1
- package/dist/utils/typeGuards.js +0 -3
- package/dist/ydb/QueryDiagnostics.d.ts +0 -6
- package/dist/ydb/QueryDiagnostics.js +0 -52
- package/dist/ydb/SessionPool.d.ts +0 -36
- package/dist/ydb/SessionPool.js +0 -248
- package/dist/ydb/bulkUpsert.d.ts +0 -6
- package/dist/ydb/bulkUpsert.js +0 -52
- /package/dist/qdrant/{QdrantTypes.js → QdrantRestTypes.js} +0 -0
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
const IS_LITTLE_ENDIAN = new Uint8Array(new Uint16Array([0x00ff]).buffer)[0] === 0xff;
|
|
1
2
|
export function vectorToFloatBinary(vector) {
|
|
2
3
|
if (vector.length === 0) {
|
|
3
4
|
return Buffer.from([1]);
|
|
@@ -8,7 +9,12 @@ export function vectorToFloatBinary(vector) {
|
|
|
8
9
|
if (!Number.isFinite(value)) {
|
|
9
10
|
throw new Error(`Non-finite value in vector at index ${i}: ${value}`);
|
|
10
11
|
}
|
|
11
|
-
|
|
12
|
+
if (IS_LITTLE_ENDIAN) {
|
|
13
|
+
buffer.writeFloatLE(value, i * 4);
|
|
14
|
+
}
|
|
15
|
+
else {
|
|
16
|
+
buffer.writeFloatBE(value, i * 4);
|
|
17
|
+
}
|
|
12
18
|
}
|
|
13
19
|
buffer.writeUInt8(1, vector.length * 4);
|
|
14
20
|
return buffer;
|
|
@@ -24,10 +30,9 @@ export function vectorToBitBinary(vector) {
|
|
|
24
30
|
const dataByteLen = Math.ceil(bitLen / 8);
|
|
25
31
|
const totalLen = dataByteLen + 2; // +1 unused-bit-count +1 format marker
|
|
26
32
|
const buffer = Buffer.alloc(totalLen);
|
|
27
|
-
const isLittleEndian = new Uint8Array(new Uint16Array([0x00ff]).buffer)[0] === 0xff;
|
|
28
33
|
let offset = 0;
|
|
29
34
|
const writeU64 = (v) => {
|
|
30
|
-
if (
|
|
35
|
+
if (IS_LITTLE_ENDIAN) {
|
|
31
36
|
buffer.writeBigUInt64LE(v, offset);
|
|
32
37
|
}
|
|
33
38
|
else {
|
|
@@ -36,7 +41,7 @@ export function vectorToBitBinary(vector) {
|
|
|
36
41
|
offset += 8;
|
|
37
42
|
};
|
|
38
43
|
const writeU32 = (v) => {
|
|
39
|
-
if (
|
|
44
|
+
if (IS_LITTLE_ENDIAN) {
|
|
40
45
|
buffer.writeUInt32LE(v, offset);
|
|
41
46
|
}
|
|
42
47
|
else {
|
|
@@ -45,7 +50,7 @@ export function vectorToBitBinary(vector) {
|
|
|
45
50
|
offset += 4;
|
|
46
51
|
};
|
|
47
52
|
const writeU16 = (v) => {
|
|
48
|
-
if (
|
|
53
|
+
if (IS_LITTLE_ENDIAN) {
|
|
49
54
|
buffer.writeUInt16LE(v, offset);
|
|
50
55
|
}
|
|
51
56
|
else {
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Creates the qdr__collections metadata table if it does not exist.
|
|
3
|
+
*
|
|
4
|
+
* Intended for CI and local dev environments where a fresh YDB is provisioned
|
|
5
|
+
* and schema creation is handled out-of-band from `ensureMetaTable()`.
|
|
6
|
+
*/
|
|
7
|
+
export declare function bootstrapMetaTable(): Promise<void>;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { Column, TableDescription, Types, destroyDriver, readyOrThrow, withSession, } from "./client.js";
|
|
2
|
+
function isTableNotFoundError(err) {
|
|
3
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
4
|
+
return (/table.*not found/i.test(msg) ||
|
|
5
|
+
/path.*not found/i.test(msg) ||
|
|
6
|
+
/does not exist/i.test(msg) ||
|
|
7
|
+
/NotFound\s*\(code\s*400140\)/i.test(msg) ||
|
|
8
|
+
(/SchemeError\s*\(code\s*400070\)/i.test(msg) &&
|
|
9
|
+
/:\s*\[\s*\]\s*$/i.test(msg)));
|
|
10
|
+
}
|
|
11
|
+
function isAlreadyExistsError(err) {
|
|
12
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
13
|
+
return /already exists/i.test(msg) || /path.*exists/i.test(msg);
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Creates the qdr__collections metadata table if it does not exist.
|
|
17
|
+
*
|
|
18
|
+
* Intended for CI and local dev environments where a fresh YDB is provisioned
|
|
19
|
+
* and schema creation is handled out-of-band from `ensureMetaTable()`.
|
|
20
|
+
*/
|
|
21
|
+
export async function bootstrapMetaTable() {
|
|
22
|
+
await readyOrThrow();
|
|
23
|
+
await withSession(async (s) => {
|
|
24
|
+
let desc = null;
|
|
25
|
+
try {
|
|
26
|
+
desc = await s.describeTable("qdr__collections");
|
|
27
|
+
}
|
|
28
|
+
catch (err) {
|
|
29
|
+
if (!isTableNotFoundError(err)) {
|
|
30
|
+
throw err;
|
|
31
|
+
}
|
|
32
|
+
const td = new TableDescription()
|
|
33
|
+
.withColumns(new Column("collection", Types.UTF8), new Column("table_name", Types.UTF8), new Column("vector_dimension", Types.UINT32), new Column("distance", Types.UTF8), new Column("vector_type", Types.UTF8), new Column("created_at", Types.TIMESTAMP), new Column("last_accessed_at", Types.TIMESTAMP))
|
|
34
|
+
.withPrimaryKey("collection");
|
|
35
|
+
try {
|
|
36
|
+
await s.createTable("qdr__collections", td);
|
|
37
|
+
console.log("bootstrapMetaTable: created qdr__collections");
|
|
38
|
+
}
|
|
39
|
+
catch (createErr) {
|
|
40
|
+
if (!isAlreadyExistsError(createErr)) {
|
|
41
|
+
throw createErr;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
desc = await s.describeTable("qdr__collections");
|
|
45
|
+
}
|
|
46
|
+
const cols = desc?.columns ?? [];
|
|
47
|
+
const hasLastAccessedAt = cols.some((c) => c.name === "last_accessed_at");
|
|
48
|
+
if (!hasLastAccessedAt) {
|
|
49
|
+
throw new Error("bootstrapMetaTable: qdr__collections exists but is missing required column last_accessed_at");
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
// CLI entrypoint
|
|
54
|
+
if (process.argv[1]?.endsWith("bootstrapMetaTable.js")) {
|
|
55
|
+
const main = async () => {
|
|
56
|
+
await bootstrapMetaTable();
|
|
57
|
+
// Important: ydb-sdk driver/session pool keeps timers alive; explicitly
|
|
58
|
+
// destroy it so this script can terminate in CI steps.
|
|
59
|
+
await destroyDriver();
|
|
60
|
+
};
|
|
61
|
+
main()
|
|
62
|
+
.then(() => {
|
|
63
|
+
process.exit(0);
|
|
64
|
+
})
|
|
65
|
+
.catch(async (err) => {
|
|
66
|
+
console.error(err);
|
|
67
|
+
try {
|
|
68
|
+
await destroyDriver();
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
// ignore cleanup errors
|
|
72
|
+
}
|
|
73
|
+
process.exit(1);
|
|
74
|
+
});
|
|
75
|
+
}
|
package/dist/ydb/client.d.ts
CHANGED
|
@@ -1,29 +1,36 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
3
|
-
|
|
1
|
+
import type { Session, IAuthService, ExecuteQuerySettings as YdbExecuteQuerySettings, BulkUpsertSettings as YdbBulkUpsertSettings, QuerySession } from "ydb-sdk";
|
|
2
|
+
declare const Types: typeof import("ydb-sdk").Types, TypedValues: typeof import("ydb-sdk").TypedValues, TableDescription: typeof import("ydb-sdk").TableDescription, Column: typeof import("ydb-sdk").Column, ExecuteQuerySettings: typeof YdbExecuteQuerySettings, BulkUpsertSettings: typeof YdbBulkUpsertSettings, Ydb: typeof import("ydb-sdk-proto").Ydb;
|
|
3
|
+
export { Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, BulkUpsertSettings, Ydb, };
|
|
4
|
+
export declare function createExecuteQuerySettings(options?: {
|
|
5
|
+
keepInCache?: boolean;
|
|
6
|
+
idempotent?: boolean;
|
|
7
|
+
}): YdbExecuteQuerySettings;
|
|
8
|
+
export declare function createExecuteQuerySettingsWithTimeout(options: {
|
|
9
|
+
keepInCache?: boolean;
|
|
10
|
+
idempotent?: boolean;
|
|
11
|
+
timeoutMs: number;
|
|
12
|
+
}): YdbExecuteQuerySettings;
|
|
13
|
+
export declare function createBulkUpsertSettingsWithTimeout(options: {
|
|
14
|
+
timeoutMs: number;
|
|
15
|
+
}): YdbBulkUpsertSettings;
|
|
4
16
|
type DriverConfig = {
|
|
5
17
|
endpoint?: string;
|
|
6
18
|
database?: string;
|
|
7
19
|
connectionString?: string;
|
|
8
|
-
|
|
20
|
+
authService?: IAuthService;
|
|
9
21
|
};
|
|
10
22
|
export declare function isCompilationTimeoutError(error: unknown): boolean;
|
|
11
|
-
export declare function isTimeoutAbortError(error: unknown): boolean;
|
|
12
|
-
export declare function getAbortErrorCause(error: unknown): unknown;
|
|
13
23
|
export declare function __setDriverForTests(fake: unknown): void;
|
|
14
|
-
export declare function
|
|
15
|
-
export declare function __setDriverFactoryForTests(factory: ((connectionString: string, options?: DriverOptions) => Driver) | undefined): void;
|
|
24
|
+
export declare function __setDriverFactoryForTests(factory: ((config: unknown) => unknown) | undefined): void;
|
|
16
25
|
export declare function __resetRefreshStateForTests(): void;
|
|
17
26
|
export declare function configureDriver(config: DriverConfig): void;
|
|
18
|
-
/**
|
|
19
|
-
* @internal
|
|
20
|
-
* Exposes the singleton YDB `Driver` for internal modules that need non-QueryService RPCs
|
|
21
|
-
* (e.g. TableService BulkUpsert).
|
|
22
|
-
*/
|
|
23
|
-
export declare function __getDriverForInternalUse(): Driver;
|
|
24
27
|
export declare function readyOrThrow(): Promise<void>;
|
|
25
|
-
export declare function withSession<T>(fn: (
|
|
26
|
-
export declare function
|
|
28
|
+
export declare function withSession<T>(fn: (s: Session) => Promise<T>): Promise<T>;
|
|
29
|
+
export declare function withQuerySession<T>(fn: (s: QuerySession) => Promise<T>, options?: {
|
|
30
|
+
timeoutMs?: number;
|
|
31
|
+
idempotent?: boolean;
|
|
32
|
+
}): Promise<T>;
|
|
33
|
+
export declare function withStartupProbeSession<T>(fn: (s: Session) => Promise<T>): Promise<T>;
|
|
27
34
|
export declare function isYdbAvailable(timeoutMs?: number): Promise<boolean>;
|
|
28
35
|
/**
|
|
29
36
|
* Destroys the current driver and its session pool.
|
|
@@ -35,4 +42,3 @@ export declare function destroyDriver(): Promise<void>;
|
|
|
35
42
|
* Use this to recover from session pool exhaustion or zombie sessions.
|
|
36
43
|
*/
|
|
37
44
|
export declare function refreshDriver(): Promise<void>;
|
|
38
|
-
export {};
|