ydb-qdrant 5.2.1 → 7.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/config/env.d.ts +9 -3
- package/dist/config/env.js +16 -5
- package/dist/package/api.d.ts +2 -2
- package/dist/package/api.js +2 -2
- package/dist/qdrant/QdrantTypes.d.ts +19 -0
- package/dist/qdrant/QdrantTypes.js +1 -0
- package/dist/repositories/collectionsRepo.d.ts +12 -7
- package/dist/repositories/collectionsRepo.js +157 -39
- package/dist/repositories/collectionsRepo.one-table.js +47 -129
- package/dist/repositories/pointsRepo.d.ts +5 -7
- package/dist/repositories/pointsRepo.js +6 -3
- package/dist/repositories/pointsRepo.one-table/Delete.d.ts +2 -0
- package/dist/repositories/pointsRepo.one-table/Delete.js +111 -0
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.d.ts +11 -0
- package/dist/repositories/pointsRepo.one-table/PathSegmentsFilter.js +32 -0
- package/dist/repositories/pointsRepo.one-table/Search/Approximate.d.ts +18 -0
- package/dist/repositories/pointsRepo.one-table/Search/Approximate.js +119 -0
- package/dist/repositories/pointsRepo.one-table/Search/Exact.d.ts +17 -0
- package/dist/repositories/pointsRepo.one-table/Search/Exact.js +101 -0
- package/dist/repositories/pointsRepo.one-table/Search/index.d.ts +8 -0
- package/dist/repositories/pointsRepo.one-table/Search/index.js +30 -0
- package/dist/repositories/pointsRepo.one-table/Upsert.d.ts +2 -0
- package/dist/repositories/pointsRepo.one-table/Upsert.js +100 -0
- package/dist/repositories/pointsRepo.one-table.d.ts +3 -13
- package/dist/repositories/pointsRepo.one-table.js +3 -403
- package/dist/routes/collections.js +61 -7
- package/dist/routes/points.js +71 -3
- package/dist/server.d.ts +1 -0
- package/dist/server.js +70 -2
- package/dist/services/CollectionService.d.ts +9 -0
- package/dist/services/CollectionService.js +13 -1
- package/dist/services/CollectionService.shared.d.ts +1 -0
- package/dist/services/CollectionService.shared.js +3 -3
- package/dist/services/PointsService.d.ts +8 -10
- package/dist/services/PointsService.js +82 -5
- package/dist/types.d.ts +85 -8
- package/dist/types.js +43 -17
- package/dist/utils/normalization.d.ts +1 -0
- package/dist/utils/normalization.js +15 -13
- package/dist/utils/retry.js +29 -19
- package/dist/utils/tenant.d.ts +2 -2
- package/dist/utils/tenant.js +21 -6
- package/dist/utils/typeGuards.d.ts +1 -0
- package/dist/utils/typeGuards.js +3 -0
- package/dist/utils/vectorBinary.js +88 -9
- package/dist/ydb/QueryDiagnostics.d.ts +6 -0
- package/dist/ydb/QueryDiagnostics.js +52 -0
- package/dist/ydb/SessionPool.d.ts +36 -0
- package/dist/ydb/SessionPool.js +248 -0
- package/dist/ydb/bulkUpsert.d.ts +6 -0
- package/dist/ydb/bulkUpsert.js +52 -0
- package/dist/ydb/client.d.ts +17 -16
- package/dist/ydb/client.js +427 -62
- package/dist/ydb/helpers.d.ts +0 -2
- package/dist/ydb/helpers.js +0 -7
- package/dist/ydb/schema.js +172 -54
- package/package.json +12 -7
- package/dist/repositories/collectionsRepo.shared.d.ts +0 -2
- package/dist/repositories/collectionsRepo.shared.js +0 -23
package/dist/ydb/client.d.ts
CHANGED
|
@@ -1,29 +1,29 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
export declare function createExecuteQuerySettings(options?: {
|
|
5
|
-
keepInCache?: boolean;
|
|
6
|
-
idempotent?: boolean;
|
|
7
|
-
}): YdbExecuteQuerySettings;
|
|
8
|
-
export declare function createExecuteQuerySettingsWithTimeout(options: {
|
|
9
|
-
keepInCache?: boolean;
|
|
10
|
-
idempotent?: boolean;
|
|
11
|
-
timeoutMs: number;
|
|
12
|
-
}): YdbExecuteQuerySettings;
|
|
1
|
+
import { Driver, type DriverOptions } from "@ydbjs/core";
|
|
2
|
+
import { type QueryClient } from "@ydbjs/query";
|
|
3
|
+
import { CredentialsProvider } from "@ydbjs/auth";
|
|
13
4
|
type DriverConfig = {
|
|
14
5
|
endpoint?: string;
|
|
15
6
|
database?: string;
|
|
16
7
|
connectionString?: string;
|
|
17
|
-
|
|
8
|
+
credentialsProvider?: CredentialsProvider;
|
|
18
9
|
};
|
|
19
10
|
export declare function isCompilationTimeoutError(error: unknown): boolean;
|
|
11
|
+
export declare function isTimeoutAbortError(error: unknown): boolean;
|
|
12
|
+
export declare function getAbortErrorCause(error: unknown): unknown;
|
|
20
13
|
export declare function __setDriverForTests(fake: unknown): void;
|
|
21
|
-
export declare function
|
|
14
|
+
export declare function __setSessionPoolForTests(fake: unknown): void;
|
|
15
|
+
export declare function __setDriverFactoryForTests(factory: ((connectionString: string, options?: DriverOptions) => Driver) | undefined): void;
|
|
22
16
|
export declare function __resetRefreshStateForTests(): void;
|
|
23
17
|
export declare function configureDriver(config: DriverConfig): void;
|
|
18
|
+
/**
|
|
19
|
+
* @internal
|
|
20
|
+
* Exposes the singleton YDB `Driver` for internal modules that need non-QueryService RPCs
|
|
21
|
+
* (e.g. TableService BulkUpsert).
|
|
22
|
+
*/
|
|
23
|
+
export declare function __getDriverForInternalUse(): Driver;
|
|
24
24
|
export declare function readyOrThrow(): Promise<void>;
|
|
25
|
-
export declare function withSession<T>(fn: (
|
|
26
|
-
export declare function withStartupProbeSession<T>(fn: (
|
|
25
|
+
export declare function withSession<T>(fn: (sql: QueryClient, signal: AbortSignal) => Promise<T>): Promise<T>;
|
|
26
|
+
export declare function withStartupProbeSession<T>(fn: (sql: QueryClient, signal: AbortSignal) => Promise<T>): Promise<T>;
|
|
27
27
|
export declare function isYdbAvailable(timeoutMs?: number): Promise<boolean>;
|
|
28
28
|
/**
|
|
29
29
|
* Destroys the current driver and its session pool.
|
|
@@ -35,3 +35,4 @@ export declare function destroyDriver(): Promise<void>;
|
|
|
35
35
|
* Use this to recover from session pool exhaustion or zombie sessions.
|
|
36
36
|
*/
|
|
37
37
|
export declare function refreshDriver(): Promise<void>;
|
|
38
|
+
export {};
|
package/dist/ydb/client.js
CHANGED
|
@@ -1,41 +1,194 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { Driver } from "@ydbjs/core";
|
|
2
|
+
import { query } from "@ydbjs/query";
|
|
3
|
+
import { CredentialsProvider } from "@ydbjs/auth";
|
|
4
|
+
import { AnonymousCredentialsProvider } from "@ydbjs/auth/anonymous";
|
|
5
|
+
import { AccessTokenCredentialsProvider } from "@ydbjs/auth/access-token";
|
|
6
|
+
import { MetadataCredentialsProvider } from "@ydbjs/auth/metadata";
|
|
7
|
+
import { readFile } from "node:fs/promises";
|
|
8
|
+
import crypto from "node:crypto";
|
|
9
|
+
import path from "node:path";
|
|
10
|
+
import { createRequire } from "node:module";
|
|
11
|
+
import { pathToFileURL } from "node:url";
|
|
12
|
+
import { YDB_DATABASE, YDB_ENDPOINT, STARTUP_PROBE_SESSION_TIMEOUT_MS, } from "../config/env.js";
|
|
3
13
|
import { logger } from "../logging/logger.js";
|
|
4
|
-
|
|
5
|
-
const { Driver, getCredentialsFromEnv, Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, OperationParams, Ydb, } = require("ydb-sdk");
|
|
6
|
-
export { Types, TypedValues, TableDescription, Column, ExecuteQuerySettings, Ydb, };
|
|
7
|
-
export function createExecuteQuerySettings(options) {
|
|
8
|
-
const { keepInCache = true, idempotent = true } = options ?? {};
|
|
9
|
-
const settings = new ExecuteQuerySettings();
|
|
10
|
-
if (keepInCache) {
|
|
11
|
-
settings.withKeepInCache(true);
|
|
12
|
-
}
|
|
13
|
-
if (idempotent) {
|
|
14
|
-
settings.withIdempotent(true);
|
|
15
|
-
}
|
|
16
|
-
return settings;
|
|
17
|
-
}
|
|
18
|
-
export function createExecuteQuerySettingsWithTimeout(options) {
|
|
19
|
-
const settings = createExecuteQuerySettings(options);
|
|
20
|
-
const op = new OperationParams();
|
|
21
|
-
const seconds = Math.max(1, Math.ceil(options.timeoutMs / 1000));
|
|
22
|
-
// Limit both overall operation processing time and cancellation time on the
|
|
23
|
-
// server side so the probe fails fast instead of hanging for the default.
|
|
24
|
-
op.withOperationTimeoutSeconds(seconds);
|
|
25
|
-
op.withCancelAfterSeconds(seconds);
|
|
26
|
-
settings.withOperationParams(op);
|
|
27
|
-
return settings;
|
|
28
|
-
}
|
|
14
|
+
import { SessionPool } from "./SessionPool.js";
|
|
29
15
|
const DRIVER_READY_TIMEOUT_MS = 15000;
|
|
30
|
-
const TABLE_SESSION_TIMEOUT_MS = 20000;
|
|
31
16
|
const YDB_HEALTHCHECK_READY_TIMEOUT_MS = 5000;
|
|
32
17
|
const DRIVER_REFRESH_COOLDOWN_MS = 30000;
|
|
18
|
+
// Bound `withSession()` so `d.ready()` / session acquire can't hang forever under YDB stalls.
|
|
19
|
+
// Matches historical table client session timeout behavior (~20s).
|
|
20
|
+
const WITH_SESSION_TIMEOUT_MS = 20000;
|
|
33
21
|
let overrideConfig;
|
|
34
22
|
let driver;
|
|
23
|
+
let sqlClient;
|
|
24
|
+
let sessionPool;
|
|
35
25
|
let lastDriverRefreshAt = 0;
|
|
36
26
|
let driverRefreshInFlight = null;
|
|
37
|
-
// Test-only: allows injecting a mock Driver factory
|
|
27
|
+
// Test-only: allows injecting a mock Driver factory.
|
|
38
28
|
let driverFactoryOverride;
|
|
29
|
+
let queryCtxPromise = null;
|
|
30
|
+
async function getQueryCtx() {
|
|
31
|
+
if (queryCtxPromise) {
|
|
32
|
+
return await queryCtxPromise;
|
|
33
|
+
}
|
|
34
|
+
const require = createRequire(import.meta.url);
|
|
35
|
+
const entry = require.resolve("@ydbjs/query");
|
|
36
|
+
const ctxPath = path.join(path.dirname(entry), "ctx.js");
|
|
37
|
+
queryCtxPromise = import(pathToFileURL(ctxPath).href);
|
|
38
|
+
return await queryCtxPromise;
|
|
39
|
+
}
|
|
40
|
+
const IAM_TOKEN_URL = "https://iam.api.cloud.yandex.net/iam/v1/tokens";
|
|
41
|
+
const SA_JWT_MAX_AGE_SECONDS = 3600;
|
|
42
|
+
const SA_TOKEN_REFRESH_SKEW_MS = 60_000;
|
|
43
|
+
// Yandex Cloud IAM tokens: lifetime does not exceed 12 hours, but the docs recommend
|
|
44
|
+
// requesting a token more often (e.g., every hour). We refresh a bit earlier as a
|
|
45
|
+
// best-effort fallback when the API response doesn't include expiresAt.
|
|
46
|
+
// Source: https://cloud.yandex.com/en/docs/iam/operations/iam-token/create-for-sa
|
|
47
|
+
const SA_TOKEN_REFRESH_FALLBACK_MS = 55 * 60_000;
|
|
48
|
+
function parseTruthyEnv(value) {
|
|
49
|
+
if (value === undefined)
|
|
50
|
+
return false;
|
|
51
|
+
const normalized = value.trim().toLowerCase();
|
|
52
|
+
if (normalized === "" ||
|
|
53
|
+
normalized === "0" ||
|
|
54
|
+
normalized === "false" ||
|
|
55
|
+
normalized === "no" ||
|
|
56
|
+
normalized === "off") {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
return true;
|
|
60
|
+
}
|
|
61
|
+
function toBase64Url(input) {
|
|
62
|
+
const buf = Buffer.isBuffer(input) ? input : Buffer.from(input);
|
|
63
|
+
return buf
|
|
64
|
+
.toString("base64")
|
|
65
|
+
.replace(/\+/g, "-")
|
|
66
|
+
.replace(/\//g, "_")
|
|
67
|
+
.replace(/=+$/g, "");
|
|
68
|
+
}
|
|
69
|
+
function jsonToBase64Url(obj) {
|
|
70
|
+
return toBase64Url(Buffer.from(JSON.stringify(obj), "utf8"));
|
|
71
|
+
}
|
|
72
|
+
function normalizeServiceAccountPrivateKeyPem(raw) {
|
|
73
|
+
const trimmed = raw.trim();
|
|
74
|
+
if (trimmed.startsWith("PLEASE DO NOT REMOVE THIS LINE!")) {
|
|
75
|
+
return trimmed.split("\n").slice(1).join("\n").trim();
|
|
76
|
+
}
|
|
77
|
+
return trimmed;
|
|
78
|
+
}
|
|
79
|
+
async function readServiceAccountKeyFile(absPath) {
|
|
80
|
+
const raw = await readFile(absPath, "utf8");
|
|
81
|
+
const parsed = JSON.parse(raw);
|
|
82
|
+
if (typeof parsed.id !== "string" ||
|
|
83
|
+
typeof parsed.service_account_id !== "string" ||
|
|
84
|
+
typeof parsed.private_key !== "string") {
|
|
85
|
+
throw new Error("Invalid service account key JSON: expected fields id, service_account_id, private_key");
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
id: parsed.id,
|
|
89
|
+
service_account_id: parsed.service_account_id,
|
|
90
|
+
private_key: parsed.private_key,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
function createServiceAccountJwt(args) {
|
|
94
|
+
// Per Yandex Cloud IAM docs, the service account JWT must be PS256 with kid
|
|
95
|
+
// and audience set to the IAM token endpoint.
|
|
96
|
+
const header = { typ: "JWT", alg: "PS256", kid: args.keyId };
|
|
97
|
+
const payload = {
|
|
98
|
+
iss: args.serviceAccountId,
|
|
99
|
+
aud: IAM_TOKEN_URL,
|
|
100
|
+
iat: args.nowSec,
|
|
101
|
+
exp: args.nowSec + SA_JWT_MAX_AGE_SECONDS,
|
|
102
|
+
};
|
|
103
|
+
const encodedHeader = jsonToBase64Url(header);
|
|
104
|
+
const encodedPayload = jsonToBase64Url(payload);
|
|
105
|
+
const signingInput = `${encodedHeader}.${encodedPayload}`;
|
|
106
|
+
const signature = crypto.sign("RSA-SHA256", Buffer.from(signingInput), {
|
|
107
|
+
key: normalizeServiceAccountPrivateKeyPem(args.privateKeyPem),
|
|
108
|
+
padding: crypto.constants.RSA_PKCS1_PSS_PADDING,
|
|
109
|
+
saltLength: 32,
|
|
110
|
+
});
|
|
111
|
+
return `${signingInput}.${toBase64Url(signature)}`;
|
|
112
|
+
}
|
|
113
|
+
async function exchangeJwtForIamToken(args) {
|
|
114
|
+
const res = await fetch(IAM_TOKEN_URL, {
|
|
115
|
+
method: "POST",
|
|
116
|
+
headers: { "content-type": "application/json" },
|
|
117
|
+
body: JSON.stringify({ jwt: args.jwt }),
|
|
118
|
+
signal: args.signal,
|
|
119
|
+
});
|
|
120
|
+
if (!res.ok) {
|
|
121
|
+
const text = await res.text().catch(() => "");
|
|
122
|
+
throw new Error(`Failed to exchange JWT for IAM token: HTTP ${res.status}${text ? `: ${text}` : ""}`);
|
|
123
|
+
}
|
|
124
|
+
const body = (await res.json());
|
|
125
|
+
const token = body.iamToken;
|
|
126
|
+
if (typeof token !== "string" || token.length === 0) {
|
|
127
|
+
throw new Error("IAM token exchange response is missing iamToken");
|
|
128
|
+
}
|
|
129
|
+
const expiresAtRaw = body.expiresAt ?? body.expires_at;
|
|
130
|
+
const expiresAtMs = typeof expiresAtRaw === "string" ? Date.parse(expiresAtRaw) : NaN;
|
|
131
|
+
return {
|
|
132
|
+
iamToken: token,
|
|
133
|
+
expiresAtMs: Number.isFinite(expiresAtMs) ? expiresAtMs : null,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
class ServiceAccountKeyFileCredentialsProvider extends CredentialsProvider {
|
|
137
|
+
keyFilePath;
|
|
138
|
+
cached = null;
|
|
139
|
+
constructor(keyFilePath) {
|
|
140
|
+
super();
|
|
141
|
+
this.keyFilePath = keyFilePath;
|
|
142
|
+
}
|
|
143
|
+
async getToken(force = false, signal) {
|
|
144
|
+
const nowMs = Date.now();
|
|
145
|
+
if (!force && this.cached) {
|
|
146
|
+
const { token, expiresAtMs, cachedAtMs } = this.cached;
|
|
147
|
+
if (expiresAtMs !== null) {
|
|
148
|
+
if (nowMs + SA_TOKEN_REFRESH_SKEW_MS < expiresAtMs) {
|
|
149
|
+
return token;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
else {
|
|
153
|
+
// If the API doesn't provide expiresAt, refresh hourly (best-effort).
|
|
154
|
+
if (nowMs - cachedAtMs < SA_TOKEN_REFRESH_FALLBACK_MS) {
|
|
155
|
+
return token;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
const key = await readServiceAccountKeyFile(this.keyFilePath);
|
|
160
|
+
const nowSec = Math.floor(nowMs / 1000);
|
|
161
|
+
const jwt = createServiceAccountJwt({
|
|
162
|
+
keyId: key.id,
|
|
163
|
+
serviceAccountId: key.service_account_id,
|
|
164
|
+
privateKeyPem: key.private_key,
|
|
165
|
+
nowSec,
|
|
166
|
+
});
|
|
167
|
+
const { iamToken, expiresAtMs } = await exchangeJwtForIamToken({
|
|
168
|
+
jwt,
|
|
169
|
+
signal,
|
|
170
|
+
});
|
|
171
|
+
this.cached = { token: iamToken, expiresAtMs, cachedAtMs: nowMs };
|
|
172
|
+
return iamToken;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
function resolveCredentialsProviderFromEnv() {
|
|
176
|
+
const saKeyPath = process.env.YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS?.trim();
|
|
177
|
+
if (saKeyPath) {
|
|
178
|
+
return new ServiceAccountKeyFileCredentialsProvider(saKeyPath);
|
|
179
|
+
}
|
|
180
|
+
if (parseTruthyEnv(process.env.YDB_METADATA_CREDENTIALS)) {
|
|
181
|
+
return new MetadataCredentialsProvider();
|
|
182
|
+
}
|
|
183
|
+
const accessToken = process.env.YDB_ACCESS_TOKEN_CREDENTIALS?.trim();
|
|
184
|
+
if (accessToken) {
|
|
185
|
+
return new AccessTokenCredentialsProvider({ token: accessToken });
|
|
186
|
+
}
|
|
187
|
+
if (parseTruthyEnv(process.env.YDB_ANONYMOUS_CREDENTIALS)) {
|
|
188
|
+
return new AnonymousCredentialsProvider();
|
|
189
|
+
}
|
|
190
|
+
throw new Error("No YDB credentials configured. Set one of: YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS, YDB_METADATA_CREDENTIALS=1, YDB_ACCESS_TOKEN_CREDENTIALS, YDB_ANONYMOUS_CREDENTIALS=1");
|
|
191
|
+
}
|
|
39
192
|
export function isCompilationTimeoutError(error) {
|
|
40
193
|
if (!(error instanceof Error)) {
|
|
41
194
|
return false;
|
|
@@ -54,6 +207,54 @@ export function isCompilationTimeoutError(error) {
|
|
|
54
207
|
}
|
|
55
208
|
return false;
|
|
56
209
|
}
|
|
210
|
+
function getErrorCause(error) {
|
|
211
|
+
return error.cause;
|
|
212
|
+
}
|
|
213
|
+
function getCauseName(cause) {
|
|
214
|
+
if (cause instanceof Error) {
|
|
215
|
+
return cause.name;
|
|
216
|
+
}
|
|
217
|
+
if (typeof cause === "object" && cause !== null) {
|
|
218
|
+
const n = cause.name;
|
|
219
|
+
if (typeof n === "string") {
|
|
220
|
+
return n;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
return undefined;
|
|
224
|
+
}
|
|
225
|
+
export function isTimeoutAbortError(error) {
|
|
226
|
+
// `@ydbjs/query` implements `.timeout(ms)` by adding `AbortSignal.timeout(ms)` to the query
|
|
227
|
+
// signal chain. When that fires, @ydbjs/retry wraps it via `@ydbjs/abortable` and we end up
|
|
228
|
+
// observing a DOMException `{ name: "AbortError" }`, often with an underlying `cause`
|
|
229
|
+
// `{ name: "TimeoutError" }`.
|
|
230
|
+
if (!(error instanceof Error)) {
|
|
231
|
+
return false;
|
|
232
|
+
}
|
|
233
|
+
if (error.name === "TimeoutError") {
|
|
234
|
+
return true;
|
|
235
|
+
}
|
|
236
|
+
if (error.name !== "AbortError") {
|
|
237
|
+
return false;
|
|
238
|
+
}
|
|
239
|
+
const cause = getErrorCause(error);
|
|
240
|
+
const causeName = getCauseName(cause);
|
|
241
|
+
if (causeName === "TimeoutError") {
|
|
242
|
+
return true;
|
|
243
|
+
}
|
|
244
|
+
// Node versions differ in how abort reasons are propagated through `AbortSignal.any(...)`.
|
|
245
|
+
// In our codebase we currently only use AbortError signals for query timeouts / cancellations,
|
|
246
|
+
// so treat an AbortError without a usable cause as timeout-like.
|
|
247
|
+
return true;
|
|
248
|
+
}
|
|
249
|
+
export function getAbortErrorCause(error) {
|
|
250
|
+
if (!(error instanceof Error)) {
|
|
251
|
+
return undefined;
|
|
252
|
+
}
|
|
253
|
+
if (error.name !== "AbortError") {
|
|
254
|
+
return undefined;
|
|
255
|
+
}
|
|
256
|
+
return getErrorCause(error);
|
|
257
|
+
}
|
|
57
258
|
function shouldTriggerDriverRefresh(error) {
|
|
58
259
|
if (!(error instanceof Error)) {
|
|
59
260
|
return false;
|
|
@@ -68,9 +269,8 @@ function shouldTriggerDriverRefresh(error) {
|
|
|
68
269
|
if (/SessionExpired|SESSION_EXPIRED|session.*expired/i.test(msg)) {
|
|
69
270
|
return true;
|
|
70
271
|
}
|
|
71
|
-
//
|
|
72
|
-
//
|
|
73
|
-
// fresh connection state.
|
|
272
|
+
// Compilation timeout is treated as a signal to refresh the driver so that
|
|
273
|
+
// subsequent attempts use a fresh connection state.
|
|
74
274
|
if (isCompilationTimeoutError(error)) {
|
|
75
275
|
return true;
|
|
76
276
|
}
|
|
@@ -106,6 +306,12 @@ async function maybeRefreshDriverOnSessionError(error) {
|
|
|
106
306
|
}
|
|
107
307
|
export function __setDriverForTests(fake) {
|
|
108
308
|
driver = fake;
|
|
309
|
+
sqlClient = undefined;
|
|
310
|
+
sessionPool = undefined;
|
|
311
|
+
queryCtxPromise = null;
|
|
312
|
+
}
|
|
313
|
+
export function __setSessionPoolForTests(fake) {
|
|
314
|
+
sessionPool = fake;
|
|
109
315
|
}
|
|
110
316
|
export function __setDriverFactoryForTests(factory) {
|
|
111
317
|
driverFactoryOverride = factory;
|
|
@@ -121,46 +327,157 @@ export function configureDriver(config) {
|
|
|
121
327
|
}
|
|
122
328
|
overrideConfig = config;
|
|
123
329
|
}
|
|
330
|
+
function buildConnectionString(args) {
|
|
331
|
+
const endpoint = args.endpoint.trim();
|
|
332
|
+
const database = args.database.trim();
|
|
333
|
+
if (!endpoint) {
|
|
334
|
+
throw new Error("YDB endpoint is empty; set YDB_ENDPOINT");
|
|
335
|
+
}
|
|
336
|
+
if (!database) {
|
|
337
|
+
throw new Error("YDB database is empty; set YDB_DATABASE");
|
|
338
|
+
}
|
|
339
|
+
const url = new URL(endpoint);
|
|
340
|
+
const dbPath = database.startsWith("/") ? database : `/${database}`;
|
|
341
|
+
url.pathname = dbPath;
|
|
342
|
+
return url.toString();
|
|
343
|
+
}
|
|
124
344
|
function getOrCreateDriver() {
|
|
125
345
|
if (driver) {
|
|
126
346
|
return driver;
|
|
127
347
|
}
|
|
128
|
-
const
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
348
|
+
const endpoint = overrideConfig?.endpoint ?? YDB_ENDPOINT;
|
|
349
|
+
const database = overrideConfig?.database ?? YDB_DATABASE;
|
|
350
|
+
let connectionString;
|
|
351
|
+
if (overrideConfig?.connectionString) {
|
|
352
|
+
connectionString = overrideConfig.connectionString;
|
|
353
|
+
}
|
|
354
|
+
else if (driverFactoryOverride) {
|
|
355
|
+
// Tests may inject a driver factory without setting env vars.
|
|
356
|
+
// If endpoint/database are available, still build a real connection string.
|
|
357
|
+
const hasEndpoint = endpoint.trim().length > 0;
|
|
358
|
+
const hasDatabase = database.trim().length > 0;
|
|
359
|
+
connectionString =
|
|
360
|
+
hasEndpoint && hasDatabase
|
|
361
|
+
? buildConnectionString({ endpoint, database })
|
|
362
|
+
: "";
|
|
363
|
+
}
|
|
364
|
+
else {
|
|
365
|
+
connectionString = buildConnectionString({ endpoint, database });
|
|
366
|
+
}
|
|
367
|
+
const credentialsProvider = overrideConfig?.credentialsProvider
|
|
368
|
+
? overrideConfig.credentialsProvider
|
|
369
|
+
: driverFactoryOverride
|
|
370
|
+
? undefined
|
|
371
|
+
: resolveCredentialsProviderFromEnv();
|
|
372
|
+
const driverOptions = {
|
|
373
|
+
...(credentialsProvider ? { credentialsProvider } : {}),
|
|
374
|
+
"ydb.sdk.ready_timeout_ms": DRIVER_READY_TIMEOUT_MS,
|
|
142
375
|
};
|
|
143
376
|
driver = driverFactoryOverride
|
|
144
|
-
? driverFactoryOverride(
|
|
145
|
-
: new Driver(
|
|
146
|
-
logger.info({
|
|
147
|
-
poolMinSize: SESSION_POOL_MIN_SIZE,
|
|
148
|
-
poolMaxSize: SESSION_POOL_MAX_SIZE,
|
|
149
|
-
keepAlivePeriodMs: SESSION_KEEPALIVE_PERIOD_MS,
|
|
150
|
-
}, "YDB driver created with session pool settings");
|
|
377
|
+
? driverFactoryOverride(connectionString, driverOptions)
|
|
378
|
+
: new Driver(connectionString, driverOptions);
|
|
379
|
+
logger.info({ hasCredentialsProvider: true, connectionStringMasked: true }, "YDB driver created");
|
|
151
380
|
return driver;
|
|
152
381
|
}
|
|
153
|
-
|
|
382
|
+
/**
|
|
383
|
+
* @internal
|
|
384
|
+
* Exposes the singleton YDB `Driver` for internal modules that need non-QueryService RPCs
|
|
385
|
+
* (e.g. TableService BulkUpsert).
|
|
386
|
+
*/
|
|
387
|
+
export function __getDriverForInternalUse() {
|
|
388
|
+
return getOrCreateDriver();
|
|
389
|
+
}
|
|
390
|
+
function getOrCreateQueryClient() {
|
|
391
|
+
if (sqlClient) {
|
|
392
|
+
return sqlClient;
|
|
393
|
+
}
|
|
154
394
|
const d = getOrCreateDriver();
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
395
|
+
sqlClient = query(d);
|
|
396
|
+
return sqlClient;
|
|
397
|
+
}
|
|
398
|
+
function getOrCreateSessionPool() {
|
|
399
|
+
if (sessionPool) {
|
|
400
|
+
return sessionPool;
|
|
158
401
|
}
|
|
402
|
+
const d = getOrCreateDriver();
|
|
403
|
+
sessionPool = new SessionPool(d);
|
|
404
|
+
// Keepalive is best-effort; don't block startup.
|
|
405
|
+
sessionPool.start();
|
|
406
|
+
return sessionPool;
|
|
407
|
+
}
|
|
408
|
+
async function withTimeoutSignal(timeoutMs, fn) {
|
|
409
|
+
const ac = new AbortController();
|
|
410
|
+
const t = setTimeout(() => ac.abort(), timeoutMs);
|
|
411
|
+
try {
|
|
412
|
+
return await fn(ac.signal);
|
|
413
|
+
}
|
|
414
|
+
finally {
|
|
415
|
+
clearTimeout(t);
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
export async function readyOrThrow() {
|
|
419
|
+
const d = getOrCreateDriver();
|
|
420
|
+
await withTimeoutSignal(DRIVER_READY_TIMEOUT_MS, async (signal) => {
|
|
421
|
+
await d.ready(signal);
|
|
422
|
+
});
|
|
159
423
|
}
|
|
160
424
|
export async function withSession(fn) {
|
|
425
|
+
const sql = getOrCreateQueryClient();
|
|
161
426
|
const d = getOrCreateDriver();
|
|
427
|
+
const pool = getOrCreateSessionPool();
|
|
162
428
|
try {
|
|
163
|
-
|
|
429
|
+
// Bound only `d.ready()` and `pool.acquire()` so stalls in driver readiness or session
|
|
430
|
+
// availability can't hang forever. We intentionally do NOT abort the user callback,
|
|
431
|
+
// because request operations can legitimately exceed this bound (e.g. multi-batch upserts).
|
|
432
|
+
await withTimeoutSignal(WITH_SESSION_TIMEOUT_MS, async (signal) => {
|
|
433
|
+
await d.ready(signal);
|
|
434
|
+
});
|
|
435
|
+
// Ensure we have a few sessions ready to reduce cold-start session churn.
|
|
436
|
+
// Best-effort: ignore failures/timeouts.
|
|
437
|
+
void withTimeoutSignal(WITH_SESSION_TIMEOUT_MS, async (signal) => {
|
|
438
|
+
await pool.warmup(signal);
|
|
439
|
+
}).catch(() => undefined);
|
|
440
|
+
// @ydbjs/query QueryClient.do() is not implemented in the currently published builds.
|
|
441
|
+
// Cancellation is cooperative: we provide an AbortSignal to `fn()` and also store it in
|
|
442
|
+
// the @ydbjs/query async-local context so query operations can observe it via `.signal(...)`.
|
|
443
|
+
// (Note: this signal is not auto-aborted by `withSession()`; query-level `.timeout(...)`
|
|
444
|
+
// is the primary guard for long-running YQL statements.)
|
|
445
|
+
const operationSignal = new AbortController().signal;
|
|
446
|
+
const ctxMod = await getQueryCtx();
|
|
447
|
+
let leased = null;
|
|
448
|
+
const MAX_ATTEMPTS = 3;
|
|
449
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt += 1) {
|
|
450
|
+
leased = await withTimeoutSignal(WITH_SESSION_TIMEOUT_MS, async (signal) => {
|
|
451
|
+
return await pool.acquire(signal);
|
|
452
|
+
});
|
|
453
|
+
try {
|
|
454
|
+
const store = {
|
|
455
|
+
nodeId: leased.nodeId,
|
|
456
|
+
sessionId: leased.sessionId,
|
|
457
|
+
signal: operationSignal,
|
|
458
|
+
};
|
|
459
|
+
const result = await ctxMod.ctx.run(store, async () => {
|
|
460
|
+
return await fn(sql, operationSignal);
|
|
461
|
+
});
|
|
462
|
+
pool.release(leased);
|
|
463
|
+
return result;
|
|
464
|
+
}
|
|
465
|
+
catch (err) {
|
|
466
|
+
const shouldRetryWithFreshSession = err instanceof Error &&
|
|
467
|
+
/BAD_SESSION|SESSION_EXPIRED|SessionExpired|No session became available/i.test(err.message ?? "");
|
|
468
|
+
if (shouldRetryWithFreshSession) {
|
|
469
|
+
await pool.discard(leased);
|
|
470
|
+
leased = null;
|
|
471
|
+
if (attempt === MAX_ATTEMPTS - 1) {
|
|
472
|
+
throw err;
|
|
473
|
+
}
|
|
474
|
+
continue;
|
|
475
|
+
}
|
|
476
|
+
pool.release(leased);
|
|
477
|
+
throw err;
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
throw new Error("withSession: exhausted attempts to acquire a healthy session");
|
|
164
481
|
}
|
|
165
482
|
catch (err) {
|
|
166
483
|
void maybeRefreshDriverOnSessionError(err);
|
|
@@ -168,19 +485,49 @@ export async function withSession(fn) {
|
|
|
168
485
|
}
|
|
169
486
|
}
|
|
170
487
|
export async function withStartupProbeSession(fn) {
|
|
488
|
+
const sql = getOrCreateQueryClient();
|
|
171
489
|
const d = getOrCreateDriver();
|
|
490
|
+
const pool = getOrCreateSessionPool();
|
|
491
|
+
const ac = new AbortController();
|
|
492
|
+
const t = setTimeout(() => ac.abort(), STARTUP_PROBE_SESSION_TIMEOUT_MS);
|
|
172
493
|
try {
|
|
173
|
-
|
|
494
|
+
// Ensure the startup probe timeout also applies to driver readiness.
|
|
495
|
+
await d.ready(ac.signal);
|
|
496
|
+
void pool.warmup(ac.signal);
|
|
497
|
+
const ctxMod = await getQueryCtx();
|
|
498
|
+
const leased = await pool.acquire(ac.signal);
|
|
499
|
+
try {
|
|
500
|
+
const store = {
|
|
501
|
+
nodeId: leased.nodeId,
|
|
502
|
+
sessionId: leased.sessionId,
|
|
503
|
+
signal: ac.signal,
|
|
504
|
+
};
|
|
505
|
+
const result = await ctxMod.ctx.run(store, async () => {
|
|
506
|
+
return await fn(sql, ac.signal);
|
|
507
|
+
});
|
|
508
|
+
pool.release(leased);
|
|
509
|
+
return result;
|
|
510
|
+
}
|
|
511
|
+
catch (err) {
|
|
512
|
+
pool.release(leased);
|
|
513
|
+
throw err;
|
|
514
|
+
}
|
|
174
515
|
}
|
|
175
516
|
catch (err) {
|
|
176
517
|
void maybeRefreshDriverOnSessionError(err);
|
|
177
518
|
throw err;
|
|
178
519
|
}
|
|
520
|
+
finally {
|
|
521
|
+
clearTimeout(t);
|
|
522
|
+
}
|
|
179
523
|
}
|
|
180
524
|
export async function isYdbAvailable(timeoutMs = YDB_HEALTHCHECK_READY_TIMEOUT_MS) {
|
|
181
525
|
const d = getOrCreateDriver();
|
|
182
526
|
try {
|
|
183
|
-
|
|
527
|
+
await withTimeoutSignal(timeoutMs, async (signal) => {
|
|
528
|
+
await d.ready(signal);
|
|
529
|
+
});
|
|
530
|
+
return true;
|
|
184
531
|
}
|
|
185
532
|
catch {
|
|
186
533
|
return false;
|
|
@@ -194,9 +541,27 @@ export async function destroyDriver() {
|
|
|
194
541
|
if (!driver) {
|
|
195
542
|
return;
|
|
196
543
|
}
|
|
197
|
-
logger.info("Destroying YDB driver
|
|
544
|
+
logger.info("Destroying YDB driver");
|
|
198
545
|
try {
|
|
199
|
-
|
|
546
|
+
if (sessionPool) {
|
|
547
|
+
try {
|
|
548
|
+
await sessionPool.close();
|
|
549
|
+
}
|
|
550
|
+
catch (err) {
|
|
551
|
+
logger.warn({ err }, "Error during session pool close (ignored)");
|
|
552
|
+
}
|
|
553
|
+
sessionPool = undefined;
|
|
554
|
+
}
|
|
555
|
+
if (sqlClient) {
|
|
556
|
+
try {
|
|
557
|
+
await sqlClient[Symbol.asyncDispose]();
|
|
558
|
+
}
|
|
559
|
+
catch (err) {
|
|
560
|
+
logger.warn({ err }, "Error during query client disposal (ignored)");
|
|
561
|
+
}
|
|
562
|
+
sqlClient = undefined;
|
|
563
|
+
}
|
|
564
|
+
driver.close();
|
|
200
565
|
}
|
|
201
566
|
catch (err) {
|
|
202
567
|
logger.warn({ err }, "Error during driver destruction (ignored)");
|
package/dist/ydb/helpers.d.ts
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
export declare function buildVectorParam(vector: number[]): import("ydb-sdk-proto").Ydb.ITypedValue;
|
|
2
|
-
export declare function buildJsonOrEmpty(payload?: Record<string, unknown>): import("ydb-sdk-proto").Ydb.ITypedValue;
|
|
3
1
|
export declare function buildVectorBinaryParams(vector: number[]): {
|
|
4
2
|
float: Buffer<ArrayBufferLike>;
|
|
5
3
|
bit: Buffer<ArrayBufferLike>;
|
package/dist/ydb/helpers.js
CHANGED
|
@@ -1,11 +1,4 @@
|
|
|
1
|
-
import { Types, TypedValues } from "./client.js";
|
|
2
1
|
import { vectorToFloatBinary, vectorToBitBinary, } from "../utils/vectorBinary.js";
|
|
3
|
-
export function buildVectorParam(vector) {
|
|
4
|
-
return TypedValues.list(Types.FLOAT, vector);
|
|
5
|
-
}
|
|
6
|
-
export function buildJsonOrEmpty(payload) {
|
|
7
|
-
return TypedValues.jsonDocument(JSON.stringify(payload ?? {}));
|
|
8
|
-
}
|
|
9
2
|
export function buildVectorBinaryParams(vector) {
|
|
10
3
|
return {
|
|
11
4
|
float: vectorToFloatBinary(vector),
|