ydb-qdrant 2.3.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -400,7 +400,7 @@ curl -X POST http://localhost:8080/collections/mycol/points/delete \
400
400
  ## Notes
401
401
  - One YDB table is created per collection; metadata is tracked in table `qdr__collections`.
402
402
  - Each collection table schema: `point_id Utf8` (PK), `embedding String` (binary), `payload JsonDocument`.
403
- - Vectors are serialized with `Knn::ToBinaryStringFloat` (or `Knn::ToBinaryStringUint8` if collection uses uint8).
403
+ - Vectors are serialized with `Knn::ToBinaryStringFloat`.
404
404
  - Search uses a single-phase top‑k over `embedding` with automatic YDB vector index (`emb_idx`) when available; falls back to table scan if missing.
405
405
  - **Vector index auto-build**: After ≥100 points upserted + 5s quiet window, a `vector_kmeans_tree` index (levels=1, clusters=128) is built automatically. Incremental updates (<100 points) skip index rebuild.
406
406
  - **Concurrency**: During index rebuilds, YDB may return transient `Aborted`/schema metadata errors. Upserts include bounded retries with backoff to handle this automatically.
package/dist/SmokeTest.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import "dotenv/config";
2
- import { createYdbQdrantClient } from "./package/Api.js";
2
+ import { createYdbQdrantClient } from "./package/api.js";
3
3
  async function main() {
4
4
  const tenant = process.env.SMOKE_TENANT ?? "smoke";
5
5
  const collection = process.env.SMOKE_COLLECTION ?? "demo";
@@ -1,6 +1,7 @@
1
1
  import type { IAuthService } from "ydb-sdk";
2
- import { createCollection as serviceCreateCollection, deleteCollection as serviceDeleteCollection, getCollection as serviceGetCollection, putCollectionIndex as servicePutCollectionIndex, upsertPoints as serviceUpsertPoints, searchPoints as serviceSearchPoints, deletePoints as serviceDeletePoints } from "../services/QdrantService.js";
3
- export { QdrantServiceError } from "../services/QdrantService.js";
2
+ import { createCollection as serviceCreateCollection, deleteCollection as serviceDeleteCollection, getCollection as serviceGetCollection, putCollectionIndex as servicePutCollectionIndex } from "../services/CollectionService.js";
3
+ import { upsertPoints as serviceUpsertPoints, searchPoints as serviceSearchPoints, deletePoints as serviceDeletePoints } from "../services/PointsService.js";
4
+ export { QdrantServiceError } from "../services/errors.js";
4
5
  export { CreateCollectionReq, UpsertPointsReq, SearchReq, DeletePointsReq, } from "../types.js";
5
6
  type CreateCollectionResult = Awaited<ReturnType<typeof serviceCreateCollection>>;
6
7
  type GetCollectionResult = Awaited<ReturnType<typeof serviceGetCollection>>;
@@ -0,0 +1,55 @@
1
+ import { readyOrThrow, configureDriver } from "../ydb/client.js";
2
+ import { ensureMetaTable } from "../ydb/schema.js";
3
+ import { createCollection as serviceCreateCollection, deleteCollection as serviceDeleteCollection, getCollection as serviceGetCollection, putCollectionIndex as servicePutCollectionIndex, } from "../services/CollectionService.js";
4
+ import { upsertPoints as serviceUpsertPoints, searchPoints as serviceSearchPoints, deletePoints as serviceDeletePoints, } from "../services/PointsService.js";
5
+ export { QdrantServiceError } from "../services/errors.js";
6
+ export { CreateCollectionReq, UpsertPointsReq, SearchReq, DeletePointsReq, } from "../types.js";
7
+ function buildTenantClient(resolveTenant) {
8
+ return {
9
+ createCollection(collection, body) {
10
+ return serviceCreateCollection({ tenant: resolveTenant(), collection }, body);
11
+ },
12
+ getCollection(collection) {
13
+ return serviceGetCollection({ tenant: resolveTenant(), collection });
14
+ },
15
+ deleteCollection(collection) {
16
+ return serviceDeleteCollection({ tenant: resolveTenant(), collection });
17
+ },
18
+ putCollectionIndex(collection) {
19
+ return servicePutCollectionIndex({ tenant: resolveTenant(), collection });
20
+ },
21
+ upsertPoints(collection, body) {
22
+ return serviceUpsertPoints({ tenant: resolveTenant(), collection }, body);
23
+ },
24
+ searchPoints(collection, body) {
25
+ return serviceSearchPoints({ tenant: resolveTenant(), collection }, body);
26
+ },
27
+ deletePoints(collection, body) {
28
+ return serviceDeletePoints({ tenant: resolveTenant(), collection }, body);
29
+ },
30
+ };
31
+ }
32
+ export async function createYdbQdrantClient(options = {}) {
33
+ if (options.endpoint !== undefined ||
34
+ options.database !== undefined ||
35
+ options.connectionString !== undefined ||
36
+ options.authService !== undefined) {
37
+ configureDriver({
38
+ endpoint: options.endpoint,
39
+ database: options.database,
40
+ connectionString: options.connectionString,
41
+ authService: options.authService,
42
+ });
43
+ }
44
+ await readyOrThrow();
45
+ await ensureMetaTable();
46
+ const defaultTenant = options.defaultTenant ?? "default";
47
+ const baseClient = buildTenantClient(() => defaultTenant);
48
+ const client = {
49
+ ...baseClient,
50
+ forTenant(tenantId) {
51
+ return buildTenantClient(() => tenantId);
52
+ },
53
+ };
54
+ return client;
55
+ }
@@ -1,4 +1,5 @@
1
1
  import { Types, TypedValues, withSession, TableDescription, Column, } from "../ydb/client.js";
2
+ import { mapDistanceToIndexParam } from "../utils/distance.js";
2
3
  export async function createCollection(metaKey, dim, distance, vectorType, tableName) {
3
4
  await withSession(async (s) => {
4
5
  const desc = new TableDescription()
@@ -103,17 +104,3 @@ export async function buildVectorIndex(tableName, dimension, distance, vectorTyp
103
104
  await rawSession.api.executeSchemeQuery(createReq);
104
105
  });
105
106
  }
106
- function mapDistanceToIndexParam(distance) {
107
- switch (distance) {
108
- case "Cosine":
109
- return "cosine";
110
- case "Dot":
111
- return "inner_product";
112
- case "Euclid":
113
- return "euclidean";
114
- case "Manhattan":
115
- return "manhattan";
116
- default:
117
- return "cosine";
118
- }
119
- }
@@ -1,10 +1,10 @@
1
- import type { VectorType, DistanceKind } from "../types";
1
+ import type { DistanceKind } from "../types";
2
2
  export declare function upsertPoints(tableName: string, points: Array<{
3
3
  id: string | number;
4
4
  vector: number[];
5
5
  payload?: Record<string, unknown>;
6
- }>, vectorType: VectorType, dimension: number): Promise<number>;
7
- export declare function searchPoints(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, vectorType: VectorType, dimension: number): Promise<Array<{
6
+ }>, dimension: number): Promise<number>;
7
+ export declare function searchPoints(tableName: string, queryVector: number[], top: number, withPayload: boolean | undefined, distance: DistanceKind, dimension: number): Promise<Array<{
8
8
  id: string;
9
9
  score: number;
10
10
  payload?: Record<string, unknown>;
@@ -3,7 +3,9 @@ import { buildJsonOrEmpty, buildVectorParam } from "../ydb/helpers.js";
3
3
  import { logger } from "../logging/logger.js";
4
4
  import { notifyUpsert } from "../indexing/IndexScheduler.js";
5
5
  import { VECTOR_INDEX_BUILD_ENABLED } from "../config/env.js";
6
- export async function upsertPoints(tableName, points, vectorType, dimension) {
6
+ import { mapDistanceToKnnFn } from "../utils/distance.js";
7
+ import { withRetry, isTransientYdbError } from "../utils/retry.js";
8
+ export async function upsertPoints(tableName, points, dimension) {
7
9
  let upserted = 0;
8
10
  await withSession(async (s) => {
9
11
  for (const p of points) {
@@ -13,64 +15,46 @@ export async function upsertPoints(tableName, points, vectorType, dimension) {
13
15
  }
14
16
  const ddl = `
15
17
  DECLARE $id AS Utf8;
16
- DECLARE $vec AS List<${vectorType === "uint8" ? "Uint8" : "Float"}>;
18
+ DECLARE $vec AS List<Float>;
17
19
  DECLARE $payload AS JsonDocument;
18
20
  UPSERT INTO ${tableName} (point_id, embedding, payload)
19
21
  VALUES (
20
22
  $id,
21
- Untag(Knn::ToBinaryString${vectorType === "uint8" ? "Uint8" : "Float"}($vec), "${vectorType === "uint8" ? "Uint8Vector" : "FloatVector"}"),
23
+ Untag(Knn::ToBinaryStringFloat($vec), "FloatVector"),
22
24
  $payload
23
25
  );
24
26
  `;
25
27
  const params = {
26
28
  $id: TypedValues.utf8(id),
27
- $vec: buildVectorParam(p.vector, vectorType),
29
+ $vec: buildVectorParam(p.vector),
28
30
  $payload: buildJsonOrEmpty(p.payload),
29
31
  };
30
- // Retry on transient schema/metadata mismatches during index rebuild
31
- const maxRetries = 6; // ~ up to ~ (0.25 + jitter) * 2^5 ≈ few seconds
32
- let attempt = 0;
33
- while (true) {
34
- try {
35
- await s.executeQuery(ddl, params);
36
- break;
37
- }
38
- catch (e) {
39
- const msg = e instanceof Error ? e.message : String(e);
40
- const isTransient = /Aborted|schema version mismatch|Table metadata loading|Failed to load metadata/i.test(msg);
41
- if (!isTransient || attempt >= maxRetries) {
42
- throw e;
43
- }
44
- const backoffMs = Math.floor(250 * Math.pow(2, attempt) + Math.random() * 100);
45
- logger.warn({ tableName, id, attempt, backoffMs }, "upsert aborted due to schema/metadata change; retrying");
46
- await new Promise((r) => setTimeout(r, backoffMs));
47
- attempt += 1;
48
- }
49
- }
32
+ await withRetry(() => s.executeQuery(ddl, params), {
33
+ isTransient: isTransientYdbError,
34
+ context: { tableName, id },
35
+ });
50
36
  upserted += 1;
51
37
  }
52
38
  });
53
- // notify scheduler for potential end-of-batch index build
54
39
  notifyUpsert(tableName, upserted);
55
- // No index rebuild; approximate search does not require it
56
40
  return upserted;
57
41
  }
58
42
  // Removed legacy index backfill helper
59
- export async function searchPoints(tableName, queryVector, top, withPayload, distance, vectorType, dimension) {
43
+ export async function searchPoints(tableName, queryVector, top, withPayload, distance, dimension) {
60
44
  if (queryVector.length !== dimension) {
61
45
  throw new Error(`Vector dimension mismatch: got ${queryVector.length}, expected ${dimension}`);
62
46
  }
63
47
  const { fn, order } = mapDistanceToKnnFn(distance);
64
48
  // Single-phase search over embedding using vector index if present
65
- const qf = buildVectorParam(queryVector, vectorType);
49
+ const qf = buildVectorParam(queryVector);
66
50
  const params = {
67
51
  $qf: qf,
68
52
  $k2: TypedValues.uint32(top),
69
53
  };
70
54
  const buildQuery = (useIndex) => `
71
- DECLARE $qf AS List<${vectorType === "uint8" ? "Uint8" : "Float"}>;
55
+ DECLARE $qf AS List<Float>;
72
56
  DECLARE $k2 AS Uint32;
73
- $qbinf = Knn::ToBinaryString${vectorType === "uint8" ? "Uint8" : "Float"}($qf);
57
+ $qbinf = Knn::ToBinaryStringFloat($qf);
74
58
  SELECT point_id, ${withPayload ? "payload, " : ""}${fn}(embedding, $qbinf) AS score
75
59
  FROM ${tableName}${useIndex ? " VIEW emb_idx" : ""}
76
60
  ORDER BY score ${order}
@@ -147,17 +131,3 @@ export async function deletePoints(tableName, ids) {
147
131
  });
148
132
  return deleted;
149
133
  }
150
- function mapDistanceToKnnFn(distance) {
151
- switch (distance) {
152
- case "Cosine":
153
- return { fn: "Knn::CosineSimilarity", order: "DESC" };
154
- case "Dot":
155
- return { fn: "Knn::InnerProductSimilarity", order: "DESC" };
156
- case "Euclid":
157
- return { fn: "Knn::EuclideanDistance", order: "ASC" };
158
- case "Manhattan":
159
- return { fn: "Knn::ManhattanDistance", order: "ASC" };
160
- default:
161
- return { fn: "Knn::CosineSimilarity", order: "DESC" };
162
- }
163
- }
@@ -1,6 +1,7 @@
1
1
  import { Router } from "express";
2
2
  import { sanitizeCollectionName, sanitizeTenantId } from "../utils/tenant.js";
3
- import { QdrantServiceError, putCollectionIndex, createCollection, getCollection, deleteCollection, } from "../services/QdrantService.js";
3
+ import { putCollectionIndex, createCollection, getCollection, deleteCollection, } from "../services/CollectionService.js";
4
+ import { QdrantServiceError } from "../services/errors.js";
4
5
  import { logger } from "../logging/logger.js";
5
6
  export const collectionsRouter = Router();
6
7
  collectionsRouter.put("/:collection/index", async (req, res) => {
@@ -1,5 +1,6 @@
1
1
  import { Router } from "express";
2
- import { QdrantServiceError, upsertPoints, searchPoints, queryPoints, deletePoints, } from "../services/QdrantService.js";
2
+ import { upsertPoints, searchPoints, queryPoints, deletePoints, } from "../services/PointsService.js";
3
+ import { QdrantServiceError } from "../services/errors.js";
3
4
  import { logger } from "../logging/logger.js";
4
5
  export const pointsRouter = Router();
5
6
  // Qdrant-compatible: PUT /collections/:collection/points (upsert)
@@ -0,0 +1,29 @@
1
+ import { type DistanceKind } from "../types.js";
2
+ export interface CollectionContextInput {
3
+ tenant: string | undefined;
4
+ collection: string;
5
+ }
6
+ export interface NormalizedCollectionContext {
7
+ tenant: string;
8
+ collection: string;
9
+ metaKey: string;
10
+ }
11
+ export declare function normalizeCollectionContext(input: CollectionContextInput): NormalizedCollectionContext;
12
+ export declare function putCollectionIndex(ctx: CollectionContextInput): Promise<{
13
+ acknowledged: boolean;
14
+ }>;
15
+ export declare function createCollection(ctx: CollectionContextInput, body: unknown): Promise<{
16
+ name: string;
17
+ tenant: string;
18
+ }>;
19
+ export declare function getCollection(ctx: CollectionContextInput): Promise<{
20
+ name: string;
21
+ vectors: {
22
+ size: number;
23
+ distance: DistanceKind;
24
+ data_type: string;
25
+ };
26
+ }>;
27
+ export declare function deleteCollection(ctx: CollectionContextInput): Promise<{
28
+ acknowledged: boolean;
29
+ }>;
@@ -0,0 +1,78 @@
1
+ import { sanitizeCollectionName, sanitizeTenantId, metaKeyFor, tableNameFor, } from "../utils/tenant.js";
2
+ import { CreateCollectionReq } from "../types.js";
3
+ import { ensureMetaTable } from "../ydb/schema.js";
4
+ import { createCollection as repoCreateCollection, deleteCollection as repoDeleteCollection, getCollectionMeta, } from "../repositories/collectionsRepo.js";
5
+ import { QdrantServiceError } from "./errors.js";
6
+ export function normalizeCollectionContext(input) {
7
+ const tenant = sanitizeTenantId(input.tenant);
8
+ const collection = sanitizeCollectionName(input.collection);
9
+ const metaKey = metaKeyFor(tenant, collection);
10
+ return { tenant, collection, metaKey };
11
+ }
12
+ export async function putCollectionIndex(ctx) {
13
+ await ensureMetaTable();
14
+ const normalized = normalizeCollectionContext(ctx);
15
+ const meta = await getCollectionMeta(normalized.metaKey);
16
+ if (!meta) {
17
+ throw new QdrantServiceError(404, {
18
+ status: "error",
19
+ error: "collection not found",
20
+ });
21
+ }
22
+ return { acknowledged: true };
23
+ }
24
+ export async function createCollection(ctx, body) {
25
+ await ensureMetaTable();
26
+ const normalized = normalizeCollectionContext(ctx);
27
+ const parsed = CreateCollectionReq.safeParse(body);
28
+ if (!parsed.success) {
29
+ throw new QdrantServiceError(400, {
30
+ status: "error",
31
+ error: parsed.error.flatten(),
32
+ });
33
+ }
34
+ const dim = parsed.data.vectors.size;
35
+ const distance = parsed.data.vectors.distance;
36
+ const vectorType = parsed.data.vectors.data_type ?? "float";
37
+ const existing = await getCollectionMeta(normalized.metaKey);
38
+ if (existing) {
39
+ if (existing.dimension === dim &&
40
+ existing.distance === distance &&
41
+ existing.vectorType === vectorType) {
42
+ return { name: normalized.collection, tenant: normalized.tenant };
43
+ }
44
+ const errorMessage = `Collection already exists with different config: dimension=${existing.dimension}, distance=${existing.distance}, type=${existing.vectorType}`;
45
+ throw new QdrantServiceError(400, {
46
+ status: "error",
47
+ error: errorMessage,
48
+ });
49
+ }
50
+ const tableName = tableNameFor(normalized.tenant, normalized.collection);
51
+ await repoCreateCollection(normalized.metaKey, dim, distance, vectorType, tableName);
52
+ return { name: normalized.collection, tenant: normalized.tenant };
53
+ }
54
+ export async function getCollection(ctx) {
55
+ await ensureMetaTable();
56
+ const normalized = normalizeCollectionContext(ctx);
57
+ const meta = await getCollectionMeta(normalized.metaKey);
58
+ if (!meta) {
59
+ throw new QdrantServiceError(404, {
60
+ status: "error",
61
+ error: "collection not found",
62
+ });
63
+ }
64
+ return {
65
+ name: normalized.collection,
66
+ vectors: {
67
+ size: meta.dimension,
68
+ distance: meta.distance,
69
+ data_type: meta.vectorType,
70
+ },
71
+ };
72
+ }
73
+ export async function deleteCollection(ctx) {
74
+ await ensureMetaTable();
75
+ const normalized = normalizeCollectionContext(ctx);
76
+ await repoDeleteCollection(normalized.metaKey);
77
+ return { acknowledged: true };
78
+ }
@@ -0,0 +1,23 @@
1
+ import { type CollectionContextInput } from "./CollectionService.js";
2
+ type PointsContextInput = CollectionContextInput;
3
+ export declare function upsertPoints(ctx: PointsContextInput, body: unknown): Promise<{
4
+ upserted: number;
5
+ }>;
6
+ export declare function searchPoints(ctx: PointsContextInput, body: unknown): Promise<{
7
+ points: Array<{
8
+ id: string;
9
+ score: number;
10
+ payload?: Record<string, unknown>;
11
+ }>;
12
+ }>;
13
+ export declare function queryPoints(ctx: PointsContextInput, body: unknown): Promise<{
14
+ points: Array<{
15
+ id: string;
16
+ score: number;
17
+ payload?: Record<string, unknown>;
18
+ }>;
19
+ }>;
20
+ export declare function deletePoints(ctx: PointsContextInput, body: unknown): Promise<{
21
+ deleted: number;
22
+ }>;
23
+ export {};
@@ -0,0 +1,125 @@
1
+ import { UpsertPointsReq, SearchReq, DeletePointsReq } from "../types.js";
2
+ import { ensureMetaTable } from "../ydb/schema.js";
3
+ import { getCollectionMeta } from "../repositories/collectionsRepo.js";
4
+ import { deletePoints as repoDeletePoints, searchPoints as repoSearchPoints, upsertPoints as repoUpsertPoints, } from "../repositories/pointsRepo.js";
5
+ import { requestIndexBuild } from "../indexing/IndexScheduler.js";
6
+ import { logger } from "../logging/logger.js";
7
+ import { VECTOR_INDEX_BUILD_ENABLED } from "../config/env.js";
8
+ import { QdrantServiceError } from "./errors.js";
9
+ import { normalizeCollectionContext, } from "./CollectionService.js";
10
+ import { normalizeSearchBodyForSearch, normalizeSearchBodyForQuery, } from "../utils/normalization.js";
11
+ let loggedIndexBuildDisabled = false;
12
+ export async function upsertPoints(ctx, body) {
13
+ await ensureMetaTable();
14
+ const normalized = normalizeCollectionContext(ctx);
15
+ const meta = await getCollectionMeta(normalized.metaKey);
16
+ if (!meta) {
17
+ throw new QdrantServiceError(404, {
18
+ status: "error",
19
+ error: "collection not found",
20
+ });
21
+ }
22
+ const parsed = UpsertPointsReq.safeParse(body);
23
+ if (!parsed.success) {
24
+ throw new QdrantServiceError(400, {
25
+ status: "error",
26
+ error: parsed.error.flatten(),
27
+ });
28
+ }
29
+ const upserted = await repoUpsertPoints(meta.table, parsed.data.points, meta.dimension);
30
+ if (VECTOR_INDEX_BUILD_ENABLED) {
31
+ requestIndexBuild(meta.table, meta.dimension, meta.distance, meta.vectorType);
32
+ }
33
+ else if (!loggedIndexBuildDisabled) {
34
+ logger.info({ table: meta.table }, "vector index building disabled by env; skipping automatic emb_idx rebuilds");
35
+ loggedIndexBuildDisabled = true;
36
+ }
37
+ return { upserted };
38
+ }
39
+ async function executeSearch(ctx, normalizedSearch, source) {
40
+ await ensureMetaTable();
41
+ const normalized = normalizeCollectionContext(ctx);
42
+ logger.info({ tenant: normalized.tenant, collection: normalized.collection }, `${source}: resolve collection meta`);
43
+ const meta = await getCollectionMeta(normalized.metaKey);
44
+ if (!meta) {
45
+ logger.warn({
46
+ tenant: normalized.tenant,
47
+ collection: normalized.collection,
48
+ metaKey: normalized.metaKey,
49
+ }, `${source}: collection not found`);
50
+ throw new QdrantServiceError(404, {
51
+ status: "error",
52
+ error: "collection not found",
53
+ });
54
+ }
55
+ const parsed = SearchReq.safeParse({
56
+ vector: normalizedSearch.vector,
57
+ top: normalizedSearch.top,
58
+ with_payload: normalizedSearch.withPayload,
59
+ });
60
+ if (!parsed.success) {
61
+ logger.warn({
62
+ tenant: normalized.tenant,
63
+ collection: normalized.collection,
64
+ issues: parsed.error.issues,
65
+ }, `${source}: invalid payload`);
66
+ throw new QdrantServiceError(400, {
67
+ status: "error",
68
+ error: parsed.error.flatten(),
69
+ });
70
+ }
71
+ logger.info({
72
+ tenant: normalized.tenant,
73
+ collection: normalized.collection,
74
+ top: parsed.data.top,
75
+ queryVectorLen: parsed.data.vector.length,
76
+ collectionDim: meta.dimension,
77
+ distance: meta.distance,
78
+ vectorType: meta.vectorType,
79
+ }, `${source}: executing`);
80
+ const hits = await repoSearchPoints(meta.table, parsed.data.vector, parsed.data.top, parsed.data.with_payload, meta.distance, meta.dimension);
81
+ const threshold = normalizedSearch.scoreThreshold;
82
+ const filtered = threshold === undefined
83
+ ? hits
84
+ : hits.filter((hit) => {
85
+ const isSimilarity = meta.distance === "Cosine" || meta.distance === "Dot";
86
+ if (isSimilarity) {
87
+ return hit.score >= threshold;
88
+ }
89
+ return hit.score <= threshold;
90
+ });
91
+ logger.info({
92
+ tenant: normalized.tenant,
93
+ collection: normalized.collection,
94
+ hits: hits.length,
95
+ }, `${source}: completed`);
96
+ return { points: filtered };
97
+ }
98
+ export async function searchPoints(ctx, body) {
99
+ const normalizedSearch = normalizeSearchBodyForSearch(body);
100
+ return await executeSearch(ctx, normalizedSearch, "search");
101
+ }
102
+ export async function queryPoints(ctx, body) {
103
+ const normalizedSearch = normalizeSearchBodyForQuery(body);
104
+ return await executeSearch(ctx, normalizedSearch, "query");
105
+ }
106
+ export async function deletePoints(ctx, body) {
107
+ await ensureMetaTable();
108
+ const normalized = normalizeCollectionContext(ctx);
109
+ const meta = await getCollectionMeta(normalized.metaKey);
110
+ if (!meta) {
111
+ throw new QdrantServiceError(404, {
112
+ status: "error",
113
+ error: "collection not found",
114
+ });
115
+ }
116
+ const parsed = DeletePointsReq.safeParse(body);
117
+ if (!parsed.success) {
118
+ throw new QdrantServiceError(400, {
119
+ status: "error",
120
+ error: parsed.error.flatten(),
121
+ });
122
+ }
123
+ const deleted = await repoDeletePoints(meta.table, parsed.data.points);
124
+ return { deleted };
125
+ }
@@ -0,0 +1,9 @@
1
+ export interface QdrantServiceErrorPayload {
2
+ status: "error";
3
+ error: unknown;
4
+ }
5
+ export declare class QdrantServiceError extends Error {
6
+ readonly statusCode: number;
7
+ readonly payload: QdrantServiceErrorPayload;
8
+ constructor(statusCode: number, payload: QdrantServiceErrorPayload, message?: string);
9
+ }
@@ -0,0 +1,9 @@
1
+ export class QdrantServiceError extends Error {
2
+ statusCode;
3
+ payload;
4
+ constructor(statusCode, payload, message) {
5
+ super(message ?? String(payload.error));
6
+ this.statusCode = statusCode;
7
+ this.payload = payload;
8
+ }
9
+ }
package/dist/types.d.ts CHANGED
@@ -1,13 +1,12 @@
1
1
  import { z } from "zod";
2
2
  export type DistanceKind = "Cosine" | "Euclid" | "Dot" | "Manhattan";
3
- export type VectorType = "float" | "uint8";
3
+ export type VectorType = "float";
4
4
  export declare const CreateCollectionReq: z.ZodObject<{
5
5
  vectors: z.ZodObject<{
6
6
  size: z.ZodNumber;
7
7
  distance: z.ZodType<DistanceKind>;
8
8
  data_type: z.ZodOptional<z.ZodEnum<{
9
9
  float: "float";
10
- uint8: "uint8";
11
10
  }>>;
12
11
  }, z.core.$strip>;
13
12
  }, z.core.$strip>;
package/dist/types.js CHANGED
@@ -8,7 +8,7 @@ export const CreateCollectionReq = z.object({
8
8
  "Dot",
9
9
  "Manhattan",
10
10
  ]),
11
- data_type: z.enum(["float", "uint8"]).optional(),
11
+ data_type: z.enum(["float"]).optional(),
12
12
  }),
13
13
  });
14
14
  export const UpsertPointsReq = z.object({
@@ -0,0 +1,6 @@
1
+ import type { DistanceKind } from "../types.js";
2
+ export declare function mapDistanceToKnnFn(distance: DistanceKind): {
3
+ fn: string;
4
+ order: "ASC" | "DESC";
5
+ };
6
+ export declare function mapDistanceToIndexParam(distance: DistanceKind): string;
@@ -0,0 +1,28 @@
1
+ export function mapDistanceToKnnFn(distance) {
2
+ switch (distance) {
3
+ case "Cosine":
4
+ return { fn: "Knn::CosineSimilarity", order: "DESC" };
5
+ case "Dot":
6
+ return { fn: "Knn::InnerProductSimilarity", order: "DESC" };
7
+ case "Euclid":
8
+ return { fn: "Knn::EuclideanDistance", order: "ASC" };
9
+ case "Manhattan":
10
+ return { fn: "Knn::ManhattanDistance", order: "ASC" };
11
+ default:
12
+ return { fn: "Knn::CosineSimilarity", order: "DESC" };
13
+ }
14
+ }
15
+ export function mapDistanceToIndexParam(distance) {
16
+ switch (distance) {
17
+ case "Cosine":
18
+ return "cosine";
19
+ case "Dot":
20
+ return "inner_product";
21
+ case "Euclid":
22
+ return "euclidean";
23
+ case "Manhattan":
24
+ return "manhattan";
25
+ default:
26
+ return "cosine";
27
+ }
28
+ }
@@ -0,0 +1,10 @@
1
+ export interface SearchNormalizationResult {
2
+ vector: number[] | undefined;
3
+ top: number | undefined;
4
+ withPayload: boolean | undefined;
5
+ scoreThreshold: number | undefined;
6
+ }
7
+ export declare function isNumberArray(value: unknown): value is number[];
8
+ export declare function extractVectorLoose(body: unknown, depth?: number): number[] | undefined;
9
+ export declare function normalizeSearchBodyForSearch(body: unknown): SearchNormalizationResult;
10
+ export declare function normalizeSearchBodyForQuery(body: unknown): SearchNormalizationResult;
@@ -0,0 +1,92 @@
1
+ export function isNumberArray(value) {
2
+ return Array.isArray(value) && value.every((x) => typeof x === "number");
3
+ }
4
+ export function extractVectorLoose(body, depth = 0) {
5
+ if (!body || typeof body !== "object" || depth > 3) {
6
+ return undefined;
7
+ }
8
+ const obj = body;
9
+ if (isNumberArray(obj.vector))
10
+ return obj.vector;
11
+ if (isNumberArray(obj.embedding))
12
+ return obj.embedding;
13
+ const query = obj.query;
14
+ if (query) {
15
+ const queryVector = query["vector"];
16
+ if (isNumberArray(queryVector))
17
+ return queryVector;
18
+ const nearest = query["nearest"];
19
+ if (nearest && isNumberArray(nearest.vector)) {
20
+ return nearest.vector;
21
+ }
22
+ }
23
+ const nearest = obj.nearest;
24
+ if (nearest && isNumberArray(nearest.vector)) {
25
+ return nearest.vector;
26
+ }
27
+ for (const key of Object.keys(obj)) {
28
+ const value = obj[key];
29
+ if (isNumberArray(value)) {
30
+ return value;
31
+ }
32
+ }
33
+ for (const key of Object.keys(obj)) {
34
+ const value = obj[key];
35
+ if (value && typeof value === "object") {
36
+ const found = extractVectorLoose(value, depth + 1);
37
+ if (found) {
38
+ return found;
39
+ }
40
+ }
41
+ }
42
+ return undefined;
43
+ }
44
+ export function normalizeSearchBodyForSearch(body) {
45
+ if (!body || typeof body !== "object") {
46
+ return {
47
+ vector: undefined,
48
+ top: undefined,
49
+ withPayload: undefined,
50
+ scoreThreshold: undefined,
51
+ };
52
+ }
53
+ const b = body;
54
+ const rawVector = b["vector"];
55
+ const vector = isNumberArray(rawVector) ? rawVector : undefined;
56
+ return normalizeSearchCommon(b, vector);
57
+ }
58
+ export function normalizeSearchBodyForQuery(body) {
59
+ if (!body || typeof body !== "object") {
60
+ return {
61
+ vector: undefined,
62
+ top: undefined,
63
+ withPayload: undefined,
64
+ scoreThreshold: undefined,
65
+ };
66
+ }
67
+ const b = body;
68
+ const vector = extractVectorLoose(b);
69
+ return normalizeSearchCommon(b, vector);
70
+ }
71
+ function normalizeSearchCommon(b, vector) {
72
+ const rawTop = b["top"];
73
+ const rawLimit = b["limit"];
74
+ const topFromTop = typeof rawTop === "number" ? rawTop : undefined;
75
+ const topFromLimit = typeof rawLimit === "number" ? rawLimit : undefined;
76
+ const top = topFromTop ?? topFromLimit;
77
+ let withPayload;
78
+ const rawWithPayload = b["with_payload"];
79
+ if (typeof rawWithPayload === "boolean") {
80
+ withPayload = rawWithPayload;
81
+ }
82
+ else if (Array.isArray(rawWithPayload) ||
83
+ typeof rawWithPayload === "object") {
84
+ withPayload = true;
85
+ }
86
+ const thresholdRaw = b["score_threshold"];
87
+ const thresholdValue = typeof thresholdRaw === "number" ? thresholdRaw : Number(thresholdRaw);
88
+ const scoreThreshold = Number.isFinite(thresholdValue)
89
+ ? thresholdValue
90
+ : undefined;
91
+ return { vector, top, withPayload, scoreThreshold };
92
+ }
@@ -0,0 +1,8 @@
1
+ export interface RetryOptions {
2
+ maxRetries?: number;
3
+ baseDelayMs?: number;
4
+ isTransient?: (error: unknown) => boolean;
5
+ context?: Record<string, unknown>;
6
+ }
7
+ export declare function isTransientYdbError(error: unknown): boolean;
8
+ export declare function withRetry<T>(fn: () => Promise<T>, options?: RetryOptions): Promise<T>;
@@ -0,0 +1,28 @@
1
+ import { logger } from "../logging/logger.js";
2
+ const DEFAULT_MAX_RETRIES = 6;
3
+ const DEFAULT_BASE_DELAY_MS = 250;
4
+ export function isTransientYdbError(error) {
5
+ const msg = error instanceof Error ? error.message : String(error);
6
+ return /Aborted|schema version mismatch|Table metadata loading|Failed to load metadata/i.test(msg);
7
+ }
8
+ export async function withRetry(fn, options = {}) {
9
+ const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
10
+ const baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;
11
+ const isTransient = options.isTransient ?? isTransientYdbError;
12
+ const context = options.context ?? {};
13
+ let attempt = 0;
14
+ while (true) {
15
+ try {
16
+ return await fn();
17
+ }
18
+ catch (e) {
19
+ if (!isTransient(e) || attempt >= maxRetries) {
20
+ throw e;
21
+ }
22
+ const backoffMs = Math.floor(baseDelayMs * Math.pow(2, attempt) + Math.random() * 100);
23
+ logger.warn({ ...context, attempt, backoffMs }, "operation aborted due to transient error; retrying");
24
+ await new Promise((r) => setTimeout(r, backoffMs));
25
+ attempt += 1;
26
+ }
27
+ }
28
+ }
@@ -1,2 +1,2 @@
1
- export declare function buildVectorParam(vector: number[], vectorType: "float" | "uint8"): import("ydb-sdk-proto").Ydb.ITypedValue;
1
+ export declare function buildVectorParam(vector: number[]): import("ydb-sdk-proto").Ydb.ITypedValue;
2
2
  export declare function buildJsonOrEmpty(payload?: Record<string, unknown>): import("ydb-sdk-proto").Ydb.ITypedValue;
@@ -1,44 +1,6 @@
1
1
  import { Types, TypedValues } from "./client.js";
2
- export function buildVectorParam(vector, vectorType) {
3
- let list;
4
- if (vectorType === "uint8") {
5
- // Check if vector is already quantized (integers in [0,255])
6
- const isAlreadyQuantized = vector.every(v => Number.isInteger(v) && v >= 0 && v <= 255);
7
- if (isAlreadyQuantized) {
8
- list = vector;
9
- }
10
- else {
11
- // Float embeddings need quantization. Per YDB docs (knn.md lines 282-294):
12
- // Formula: ((x - min) / (max - min)) * 255
13
- const min = Math.min(...vector);
14
- const max = Math.max(...vector);
15
- // Determine quantization strategy based on detected range
16
- if (min >= 0 && max <= 1.01) {
17
- // Normalized [0,1] embeddings (common for some models)
18
- list = vector.map(v => Math.round(Math.max(0, Math.min(1, v)) * 255));
19
- }
20
- else if (min >= -1.01 && max <= 1.01) {
21
- // Normalized [-1,1] embeddings (most common)
22
- // Map to [0,255]: ((x + 1) / 2) * 255 = (x + 1) * 127.5
23
- list = vector.map(v => Math.round((Math.max(-1, Math.min(1, v)) + 1) * 127.5));
24
- }
25
- else {
26
- // General case: linear scaling from [min,max] to [0,255]
27
- const range = max - min;
28
- if (range > 0) {
29
- list = vector.map(v => Math.round(((v - min) / range) * 255));
30
- }
31
- else {
32
- // All values identical; map to midpoint
33
- list = vector.map(() => 127);
34
- }
35
- }
36
- }
37
- }
38
- else {
39
- list = vector;
40
- }
41
- return TypedValues.list(vectorType === "uint8" ? Types.UINT8 : Types.FLOAT, list);
2
+ export function buildVectorParam(vector) {
3
+ return TypedValues.list(Types.FLOAT, vector);
42
4
  }
43
5
  export function buildJsonOrEmpty(payload) {
44
6
  return TypedValues.jsonDocument(JSON.stringify(payload ?? {}));
package/package.json CHANGED
@@ -1,10 +1,10 @@
1
1
  {
2
2
  "name": "ydb-qdrant",
3
- "version": "2.3.0",
4
- "main": "dist/package/Api.js",
5
- "types": "dist/package/Api.d.ts",
3
+ "version": "4.0.0",
4
+ "main": "dist/package/api.js",
5
+ "types": "dist/package/api.d.ts",
6
6
  "exports": {
7
- ".": "./dist/package/Api.js",
7
+ ".": "./dist/package/api.js",
8
8
  "./server": "./dist/server.js"
9
9
  },
10
10
  "files": [
@@ -46,7 +46,7 @@
46
46
  ],
47
47
  "author": "",
48
48
  "license": "ISC",
49
- "description": "Qdrant-compatible Node.js/TypeScript API that stores/searches embeddings in YDB using approximate coarse-to-fine vector search (quantized uint8 preselect + float refine).",
49
+ "description": "Qdrant-compatible Node.js/TypeScript API that stores/searches embeddings in YDB using single-phase top-k vector search with an automatic vector index and table-scan fallback.",
50
50
  "type": "module",
51
51
  "publishConfig": {
52
52
  "access": "public"
@@ -1,79 +0,0 @@
1
- import { readyOrThrow, configureDriver } from "../ydb/client.js";
2
- import { ensureMetaTable } from "../ydb/schema.js";
3
- import { createCollection as serviceCreateCollection, deleteCollection as serviceDeleteCollection, getCollection as serviceGetCollection, putCollectionIndex as servicePutCollectionIndex, upsertPoints as serviceUpsertPoints, searchPoints as serviceSearchPoints, deletePoints as serviceDeletePoints, } from "../services/QdrantService.js";
4
- export { QdrantServiceError } from "../services/QdrantService.js";
5
- export { CreateCollectionReq, UpsertPointsReq, SearchReq, DeletePointsReq, } from "../types.js";
6
- export async function createYdbQdrantClient(options = {}) {
7
- if (options.endpoint !== undefined ||
8
- options.database !== undefined ||
9
- options.connectionString !== undefined ||
10
- options.authService !== undefined) {
11
- configureDriver({
12
- endpoint: options.endpoint,
13
- database: options.database,
14
- connectionString: options.connectionString,
15
- authService: options.authService,
16
- });
17
- }
18
- await readyOrThrow();
19
- await ensureMetaTable();
20
- const defaultTenant = options.defaultTenant ?? "default";
21
- const resolveTenant = (tenant) => tenant ?? defaultTenant;
22
- const client = {
23
- async createCollection(collection, body) {
24
- const tenant = resolveTenant(undefined);
25
- return await serviceCreateCollection({ tenant, collection }, body);
26
- },
27
- async getCollection(collection) {
28
- const tenant = resolveTenant(undefined);
29
- return await serviceGetCollection({ tenant, collection });
30
- },
31
- async deleteCollection(collection) {
32
- const tenant = resolveTenant(undefined);
33
- return await serviceDeleteCollection({ tenant, collection });
34
- },
35
- async putCollectionIndex(collection) {
36
- const tenant = resolveTenant(undefined);
37
- return await servicePutCollectionIndex({ tenant, collection });
38
- },
39
- async upsertPoints(collection, body) {
40
- const tenant = resolveTenant(undefined);
41
- return await serviceUpsertPoints({ tenant, collection }, body);
42
- },
43
- async searchPoints(collection, body) {
44
- const tenant = resolveTenant(undefined);
45
- return await serviceSearchPoints({ tenant, collection }, body);
46
- },
47
- async deletePoints(collection, body) {
48
- const tenant = resolveTenant(undefined);
49
- return await serviceDeletePoints({ tenant, collection }, body);
50
- },
51
- forTenant(tenantId) {
52
- const tenant = tenantId;
53
- return {
54
- createCollection(collection, body) {
55
- return serviceCreateCollection({ tenant, collection }, body);
56
- },
57
- getCollection(collection) {
58
- return serviceGetCollection({ tenant, collection });
59
- },
60
- deleteCollection(collection) {
61
- return serviceDeleteCollection({ tenant, collection });
62
- },
63
- putCollectionIndex(collection) {
64
- return servicePutCollectionIndex({ tenant, collection });
65
- },
66
- upsertPoints(collection, body) {
67
- return serviceUpsertPoints({ tenant, collection }, body);
68
- },
69
- searchPoints(collection, body) {
70
- return serviceSearchPoints({ tenant, collection }, body);
71
- },
72
- deletePoints(collection, body) {
73
- return serviceDeletePoints({ tenant, collection }, body);
74
- },
75
- };
76
- },
77
- };
78
- return client;
79
- }
@@ -1,54 +0,0 @@
1
- import { type DistanceKind } from "../types.js";
2
- export interface QdrantServiceErrorPayload {
3
- status: "error";
4
- error: unknown;
5
- }
6
- export declare class QdrantServiceError extends Error {
7
- readonly statusCode: number;
8
- readonly payload: QdrantServiceErrorPayload;
9
- constructor(statusCode: number, payload: QdrantServiceErrorPayload, message?: string);
10
- }
11
- interface CollectionContextInput {
12
- tenant: string | undefined;
13
- collection: string;
14
- }
15
- export declare function putCollectionIndex(ctx: CollectionContextInput): Promise<{
16
- acknowledged: boolean;
17
- }>;
18
- export declare function createCollection(ctx: CollectionContextInput, body: unknown): Promise<{
19
- name: string;
20
- tenant: string;
21
- }>;
22
- export declare function getCollection(ctx: CollectionContextInput): Promise<{
23
- name: string;
24
- vectors: {
25
- size: number;
26
- distance: DistanceKind;
27
- data_type: string;
28
- };
29
- }>;
30
- export declare function deleteCollection(ctx: CollectionContextInput): Promise<{
31
- acknowledged: boolean;
32
- }>;
33
- type PointsContextInput = CollectionContextInput;
34
- export declare function upsertPoints(ctx: PointsContextInput, body: unknown): Promise<{
35
- upserted: number;
36
- }>;
37
- export declare function searchPoints(ctx: PointsContextInput, body: unknown): Promise<{
38
- points: Array<{
39
- id: string;
40
- score: number;
41
- payload?: Record<string, unknown>;
42
- }>;
43
- }>;
44
- export declare function queryPoints(ctx: PointsContextInput, body: unknown): Promise<{
45
- points: Array<{
46
- id: string;
47
- score: number;
48
- payload?: Record<string, unknown>;
49
- }>;
50
- }>;
51
- export declare function deletePoints(ctx: PointsContextInput, body: unknown): Promise<{
52
- deleted: number;
53
- }>;
54
- export {};
@@ -1,313 +0,0 @@
1
- import { sanitizeCollectionName, sanitizeTenantId, metaKeyFor, tableNameFor, } from "../utils/tenant.js";
2
- import { CreateCollectionReq, DeletePointsReq, SearchReq, UpsertPointsReq, } from "../types.js";
3
- import { ensureMetaTable } from "../ydb/schema.js";
4
- import { createCollection as repoCreateCollection, deleteCollection as repoDeleteCollection, getCollectionMeta, } from "../repositories/collectionsRepo.js";
5
- import { deletePoints as repoDeletePoints, searchPoints as repoSearchPoints, upsertPoints as repoUpsertPoints, } from "../repositories/pointsRepo.js";
6
- import { requestIndexBuild } from "../indexing/IndexScheduler.js";
7
- import { logger } from "../logging/logger.js";
8
- import { VECTOR_INDEX_BUILD_ENABLED } from "../config/env.js";
9
- export class QdrantServiceError extends Error {
10
- statusCode;
11
- payload;
12
- constructor(statusCode, payload, message) {
13
- super(message ?? String(payload.error));
14
- this.statusCode = statusCode;
15
- this.payload = payload;
16
- }
17
- }
18
- function normalizeCollectionContext(input) {
19
- const tenant = sanitizeTenantId(input.tenant);
20
- const collection = sanitizeCollectionName(input.collection);
21
- const metaKey = metaKeyFor(tenant, collection);
22
- return { tenant, collection, metaKey };
23
- }
24
- export async function putCollectionIndex(ctx) {
25
- await ensureMetaTable();
26
- const normalized = normalizeCollectionContext(ctx);
27
- const meta = await getCollectionMeta(normalized.metaKey);
28
- if (!meta) {
29
- throw new QdrantServiceError(404, {
30
- status: "error",
31
- error: "collection not found",
32
- });
33
- }
34
- return { acknowledged: true };
35
- }
36
- export async function createCollection(ctx, body) {
37
- await ensureMetaTable();
38
- const normalized = normalizeCollectionContext(ctx);
39
- const parsed = CreateCollectionReq.safeParse(body);
40
- if (!parsed.success) {
41
- throw new QdrantServiceError(400, {
42
- status: "error",
43
- error: parsed.error.flatten(),
44
- });
45
- }
46
- const dim = parsed.data.vectors.size;
47
- const distance = parsed.data.vectors.distance;
48
- const vectorType = parsed.data.vectors.data_type ?? "float";
49
- const existing = await getCollectionMeta(normalized.metaKey);
50
- if (existing) {
51
- if (existing.dimension === dim &&
52
- existing.distance === distance &&
53
- existing.vectorType === vectorType) {
54
- return { name: normalized.collection, tenant: normalized.tenant };
55
- }
56
- const errorMessage = `Collection already exists with different config: dimension=${existing.dimension}, distance=${existing.distance}, type=${existing.vectorType}`;
57
- throw new QdrantServiceError(400, {
58
- status: "error",
59
- error: errorMessage,
60
- });
61
- }
62
- const tableName = tableNameFor(normalized.tenant, normalized.collection);
63
- await repoCreateCollection(normalized.metaKey, dim, distance, vectorType, tableName);
64
- return { name: normalized.collection, tenant: normalized.tenant };
65
- }
66
- export async function getCollection(ctx) {
67
- await ensureMetaTable();
68
- const normalized = normalizeCollectionContext(ctx);
69
- const meta = await getCollectionMeta(normalized.metaKey);
70
- if (!meta) {
71
- throw new QdrantServiceError(404, {
72
- status: "error",
73
- error: "collection not found",
74
- });
75
- }
76
- return {
77
- name: normalized.collection,
78
- vectors: {
79
- size: meta.dimension,
80
- distance: meta.distance,
81
- data_type: meta.vectorType,
82
- },
83
- };
84
- }
85
- export async function deleteCollection(ctx) {
86
- await ensureMetaTable();
87
- const normalized = normalizeCollectionContext(ctx);
88
- await repoDeleteCollection(normalized.metaKey);
89
- return { acknowledged: true };
90
- }
91
- let loggedIndexBuildDisabled = false;
92
- function isNumberArray(value) {
93
- return Array.isArray(value) && value.every((x) => typeof x === "number");
94
- }
95
- function extractVectorLoose(body, depth = 0) {
96
- if (!body || typeof body !== "object" || depth > 3) {
97
- return undefined;
98
- }
99
- const obj = body;
100
- if (isNumberArray(obj.vector))
101
- return obj.vector;
102
- if (isNumberArray(obj.embedding))
103
- return obj.embedding;
104
- const query = obj.query;
105
- if (query) {
106
- const queryVector = query["vector"];
107
- if (isNumberArray(queryVector))
108
- return queryVector;
109
- const nearest = query["nearest"];
110
- if (nearest && isNumberArray(nearest.vector)) {
111
- return nearest.vector;
112
- }
113
- }
114
- const nearest = obj.nearest;
115
- if (nearest && isNumberArray(nearest.vector)) {
116
- return nearest.vector;
117
- }
118
- for (const key of Object.keys(obj)) {
119
- const value = obj[key];
120
- if (isNumberArray(value)) {
121
- return value;
122
- }
123
- }
124
- for (const key of Object.keys(obj)) {
125
- const value = obj[key];
126
- if (value && typeof value === "object") {
127
- const found = extractVectorLoose(value, depth + 1);
128
- if (found) {
129
- return found;
130
- }
131
- }
132
- }
133
- return undefined;
134
- }
135
- function normalizeSearchBodyForSearch(body) {
136
- if (!body || typeof body !== "object") {
137
- return {
138
- vector: undefined,
139
- top: undefined,
140
- withPayload: undefined,
141
- scoreThreshold: undefined,
142
- };
143
- }
144
- const b = body;
145
- const rawVector = b["vector"];
146
- const vector = isNumberArray(rawVector) ? rawVector : undefined;
147
- const rawTop = b["top"];
148
- const rawLimit = b["limit"];
149
- const topFromTop = typeof rawTop === "number" ? rawTop : undefined;
150
- const topFromLimit = typeof rawLimit === "number" ? rawLimit : undefined;
151
- const top = topFromTop ?? topFromLimit;
152
- let withPayload;
153
- const rawWithPayload = b["with_payload"];
154
- if (typeof rawWithPayload === "boolean") {
155
- withPayload = rawWithPayload;
156
- }
157
- else if (Array.isArray(rawWithPayload) ||
158
- typeof rawWithPayload === "object") {
159
- withPayload = true;
160
- }
161
- const thresholdRaw = b["score_threshold"];
162
- const thresholdValue = typeof thresholdRaw === "number" ? thresholdRaw : Number(thresholdRaw);
163
- const scoreThreshold = Number.isFinite(thresholdValue)
164
- ? thresholdValue
165
- : undefined;
166
- return { vector, top, withPayload, scoreThreshold };
167
- }
168
- function normalizeSearchBodyForQuery(body) {
169
- if (!body || typeof body !== "object") {
170
- return {
171
- vector: undefined,
172
- top: undefined,
173
- withPayload: undefined,
174
- scoreThreshold: undefined,
175
- };
176
- }
177
- const b = body;
178
- const vector = extractVectorLoose(b);
179
- const rawTop = b["top"];
180
- const rawLimit = b["limit"];
181
- const topFromTop = typeof rawTop === "number" ? rawTop : undefined;
182
- const topFromLimit = typeof rawLimit === "number" ? rawLimit : undefined;
183
- const top = topFromTop ?? topFromLimit;
184
- let withPayload;
185
- const rawWithPayload = b["with_payload"];
186
- if (typeof rawWithPayload === "boolean") {
187
- withPayload = rawWithPayload;
188
- }
189
- else if (Array.isArray(rawWithPayload) ||
190
- typeof rawWithPayload === "object") {
191
- withPayload = true;
192
- }
193
- const thresholdRaw = b["score_threshold"];
194
- const thresholdValue = typeof thresholdRaw === "number" ? thresholdRaw : Number(thresholdRaw);
195
- const scoreThreshold = Number.isFinite(thresholdValue)
196
- ? thresholdValue
197
- : undefined;
198
- return { vector, top, withPayload, scoreThreshold };
199
- }
200
- export async function upsertPoints(ctx, body) {
201
- await ensureMetaTable();
202
- const normalized = normalizeCollectionContext(ctx);
203
- const meta = await getCollectionMeta(normalized.metaKey);
204
- if (!meta) {
205
- throw new QdrantServiceError(404, {
206
- status: "error",
207
- error: "collection not found",
208
- });
209
- }
210
- const parsed = UpsertPointsReq.safeParse(body);
211
- if (!parsed.success) {
212
- throw new QdrantServiceError(400, {
213
- status: "error",
214
- error: parsed.error.flatten(),
215
- });
216
- }
217
- const upserted = await repoUpsertPoints(meta.table, parsed.data.points, meta.vectorType, meta.dimension);
218
- if (VECTOR_INDEX_BUILD_ENABLED) {
219
- requestIndexBuild(meta.table, meta.dimension, meta.distance, meta.vectorType);
220
- }
221
- else if (!loggedIndexBuildDisabled) {
222
- logger.info({ table: meta.table }, "vector index building disabled by env; skipping automatic emb_idx rebuilds");
223
- loggedIndexBuildDisabled = true;
224
- }
225
- return { upserted };
226
- }
227
- async function executeSearch(ctx, normalizedSearch, source) {
228
- await ensureMetaTable();
229
- const normalized = normalizeCollectionContext(ctx);
230
- logger.info({ tenant: normalized.tenant, collection: normalized.collection }, `${source}: resolve collection meta`);
231
- const meta = await getCollectionMeta(normalized.metaKey);
232
- if (!meta) {
233
- logger.warn({
234
- tenant: normalized.tenant,
235
- collection: normalized.collection,
236
- metaKey: normalized.metaKey,
237
- }, `${source}: collection not found`);
238
- throw new QdrantServiceError(404, {
239
- status: "error",
240
- error: "collection not found",
241
- });
242
- }
243
- const parsed = SearchReq.safeParse({
244
- vector: normalizedSearch.vector,
245
- top: normalizedSearch.top,
246
- with_payload: normalizedSearch.withPayload,
247
- });
248
- if (!parsed.success) {
249
- logger.warn({
250
- tenant: normalized.tenant,
251
- collection: normalized.collection,
252
- issues: parsed.error.issues,
253
- }, `${source}: invalid payload`);
254
- throw new QdrantServiceError(400, {
255
- status: "error",
256
- error: parsed.error.flatten(),
257
- });
258
- }
259
- logger.info({
260
- tenant: normalized.tenant,
261
- collection: normalized.collection,
262
- top: parsed.data.top,
263
- queryVectorLen: parsed.data.vector.length,
264
- collectionDim: meta.dimension,
265
- distance: meta.distance,
266
- vectorType: meta.vectorType,
267
- }, `${source}: executing`);
268
- const hits = await repoSearchPoints(meta.table, parsed.data.vector, parsed.data.top, parsed.data.with_payload, meta.distance, meta.vectorType, meta.dimension);
269
- const threshold = normalizedSearch.scoreThreshold;
270
- const filtered = threshold === undefined
271
- ? hits
272
- : hits.filter((hit) => {
273
- const isSimilarity = meta.distance === "Cosine" || meta.distance === "Dot";
274
- if (isSimilarity) {
275
- return hit.score >= threshold;
276
- }
277
- return hit.score <= threshold;
278
- });
279
- logger.info({
280
- tenant: normalized.tenant,
281
- collection: normalized.collection,
282
- hits: hits.length,
283
- }, `${source}: completed`);
284
- return { points: filtered };
285
- }
286
- export async function searchPoints(ctx, body) {
287
- const normalizedSearch = normalizeSearchBodyForSearch(body);
288
- return await executeSearch(ctx, normalizedSearch, "search");
289
- }
290
- export async function queryPoints(ctx, body) {
291
- const normalizedSearch = normalizeSearchBodyForQuery(body);
292
- return await executeSearch(ctx, normalizedSearch, "query");
293
- }
294
- export async function deletePoints(ctx, body) {
295
- await ensureMetaTable();
296
- const normalized = normalizeCollectionContext(ctx);
297
- const meta = await getCollectionMeta(normalized.metaKey);
298
- if (!meta) {
299
- throw new QdrantServiceError(404, {
300
- status: "error",
301
- error: "collection not found",
302
- });
303
- }
304
- const parsed = DeletePointsReq.safeParse(body);
305
- if (!parsed.success) {
306
- throw new QdrantServiceError(400, {
307
- status: "error",
308
- error: parsed.error.flatten(),
309
- });
310
- }
311
- const deleted = await repoDeletePoints(meta.table, parsed.data.points);
312
- return { deleted };
313
- }