vectordb-bench 0.0.29__py3-none-any.whl → 0.0.30__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. vectordb_bench/backend/clients/__init__.py +16 -0
  2. vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +180 -15
  3. vectordb_bench/backend/clients/aws_opensearch/cli.py +51 -21
  4. vectordb_bench/backend/clients/aws_opensearch/config.py +37 -14
  5. vectordb_bench/backend/clients/lancedb/cli.py +62 -8
  6. vectordb_bench/backend/clients/lancedb/config.py +14 -1
  7. vectordb_bench/backend/clients/lancedb/lancedb.py +21 -9
  8. vectordb_bench/backend/clients/memorydb/memorydb.py +2 -2
  9. vectordb_bench/backend/clients/milvus/cli.py +30 -9
  10. vectordb_bench/backend/clients/milvus/config.py +2 -0
  11. vectordb_bench/backend/clients/milvus/milvus.py +7 -1
  12. vectordb_bench/backend/clients/qdrant_local/cli.py +60 -0
  13. vectordb_bench/backend/clients/qdrant_local/config.py +47 -0
  14. vectordb_bench/backend/clients/qdrant_local/qdrant_local.py +232 -0
  15. vectordb_bench/backend/clients/weaviate_cloud/cli.py +29 -3
  16. vectordb_bench/backend/clients/weaviate_cloud/config.py +2 -0
  17. vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +5 -0
  18. vectordb_bench/cli/batch_cli.py +121 -0
  19. vectordb_bench/cli/vectordbbench.py +4 -0
  20. vectordb_bench/config-files/batch_sample_config.yml +17 -0
  21. vectordb_bench/frontend/config/dbCaseConfigs.py +113 -1
  22. vectordb_bench/models.py +7 -0
  23. {vectordb_bench-0.0.29.dist-info → vectordb_bench-0.0.30.dist-info}/METADATA +48 -2
  24. {vectordb_bench-0.0.29.dist-info → vectordb_bench-0.0.30.dist-info}/RECORD +28 -23
  25. {vectordb_bench-0.0.29.dist-info → vectordb_bench-0.0.30.dist-info}/WHEEL +1 -1
  26. {vectordb_bench-0.0.29.dist-info → vectordb_bench-0.0.30.dist-info}/entry_points.txt +0 -0
  27. {vectordb_bench-0.0.29.dist-info → vectordb_bench-0.0.30.dist-info}/licenses/LICENSE +0 -0
  28. {vectordb_bench-0.0.29.dist-info → vectordb_bench-0.0.30.dist-info}/top_level.txt +0 -0
@@ -25,6 +25,7 @@ class LanceDBIndexConfig(BaseModel, DBCaseConfig):
25
25
  nbits: int = 8 # Must be 4 or 8
26
26
  sample_rate: int = 256
27
27
  max_iterations: int = 50
28
+ nprobes: int = 0
28
29
 
29
30
  def index_param(self) -> dict:
30
31
  if self.index not in [
@@ -52,7 +53,11 @@ class LanceDBIndexConfig(BaseModel, DBCaseConfig):
52
53
  return params
53
54
 
54
55
  def search_param(self) -> dict:
55
- pass
56
+ params = {}
57
+ if self.nprobes > 0:
58
+ params["nprobes"] = self.nprobes
59
+
60
+ return params
56
61
 
57
62
  def parse_metric(self) -> str:
58
63
  if self.metric_type in [MetricType.L2, MetricType.COSINE]:
@@ -81,6 +86,7 @@ class LanceDBHNSWIndexConfig(LanceDBIndexConfig):
81
86
  index: IndexType = IndexType.HNSW
82
87
  m: int = 0
83
88
  ef_construction: int = 0
89
+ ef: int = 0
84
90
 
85
91
  def index_param(self) -> dict:
86
92
  params = LanceDBIndexConfig.index_param(self)
@@ -94,6 +100,13 @@ class LanceDBHNSWIndexConfig(LanceDBIndexConfig):
94
100
 
95
101
  return params
96
102
 
103
+ def search_param(self) -> dict:
104
+ params = {}
105
+ if self.ef != 0:
106
+ params = {"ef": self.ef}
107
+
108
+ return params
109
+
97
110
 
98
111
  _lancedb_case_config = {
99
112
  IndexType.IVFPQ: LanceDBIndexConfig,
@@ -32,6 +32,10 @@ class LanceDB(VectorDB):
32
32
  self.table_name = collection_name
33
33
  self.dim = dim
34
34
  self.uri = db_config["uri"]
35
+ # avoid the search_param being called every time during the search process
36
+ self.search_config = db_case_config.search_param()
37
+
38
+ log.info(f"Search config: {self.search_config}")
35
39
 
36
40
  db = lancedb.connect(self.uri)
37
41
 
@@ -45,7 +49,7 @@ class LanceDB(VectorDB):
45
49
  db.open_table(self.table_name)
46
50
  except Exception:
47
51
  schema = pa.schema(
48
- [pa.field("id", pa.int64()), pa.field("vector", pa.list_(pa.float64(), list_size=self.dim))]
52
+ [pa.field("id", pa.int64()), pa.field("vector", pa.list_(pa.float32(), list_size=self.dim))]
49
53
  )
50
54
  db.create_table(self.table_name, schema=schema, mode="overwrite")
51
55
 
@@ -77,20 +81,28 @@ class LanceDB(VectorDB):
77
81
  filters: dict | None = None,
78
82
  ) -> list[int]:
79
83
  if filters:
80
- results = (
81
- self.table.search(query)
82
- .select(["id"])
83
- .where(f"id >= {filters['id']}", prefilter=True)
84
- .limit(k)
85
- .to_list()
86
- )
84
+ results = self.table.search(query).select(["id"]).where(f"id >= {filters['id']}", prefilter=True).limit(k)
85
+ if self.case_config.index == IndexType.IVFPQ and "nprobes" in self.search_config:
86
+ results = results.nprobes(self.search_config["nprobes"]).to_list()
87
+ elif self.case_config.index == IndexType.HNSW and "ef" in self.search_config:
88
+ results = results.ef(self.search_config["ef"]).to_list()
89
+ else:
90
+ results = results.to_list()
87
91
  else:
88
- results = self.table.search(query).select(["id"]).limit(k).to_list()
92
+ results = self.table.search(query).select(["id"]).limit(k)
93
+ if self.case_config.index == IndexType.IVFPQ and "nprobes" in self.search_config:
94
+ results = results.nprobes(self.search_config["nprobes"]).to_list()
95
+ elif self.case_config.index == IndexType.HNSW and "ef" in self.search_config:
96
+ results = results.ef(self.search_config["ef"]).to_list()
97
+ else:
98
+ results = results.to_list()
99
+
89
100
  return [int(result["id"]) for result in results]
90
101
 
91
102
  def optimize(self, data_size: int | None = None):
92
103
  if self.table and hasattr(self, "case_config") and self.case_config.index != IndexType.NONE:
93
104
  log.info(f"Creating index for LanceDB table ({self.table_name})")
105
+ log.info(f"Index parameters: {self.case_config.index_param()}")
94
106
  self.table.create_index(**self.case_config.index_param())
95
107
  # Better recall with IVF_PQ (though still bad) but breaks HNSW: https://github.com/lancedb/lancedb/issues/2369
96
108
  if self.case_config.index in (IndexType.IVFPQ, IndexType.AUTOINDEX):
@@ -9,10 +9,10 @@ import redis
9
9
  from redis import Redis
10
10
  from redis.cluster import RedisCluster
11
11
  from redis.commands.search.field import NumericField, TagField, VectorField
12
- from redis.commands.search.indexDefinition import IndexDefinition
12
+ from redis.commands.search.indexDefinition import IndexDefinition, IndexType
13
13
  from redis.commands.search.query import Query
14
14
 
15
- from ..api import IndexType, VectorDB
15
+ from ..api import VectorDB
16
16
  from .config import MemoryDBIndexConfig
17
17
 
18
18
  log = logging.getLogger(__name__)
@@ -29,6 +29,17 @@ class MilvusTypedDict(TypedDict):
29
29
  str | None,
30
30
  click.option("--password", type=str, help="Db password", required=False),
31
31
  ]
32
+ num_shards: Annotated[
33
+ int,
34
+ click.option(
35
+ "--num-shards",
36
+ type=int,
37
+ help="Number of shards",
38
+ required=False,
39
+ default=1,
40
+ show_default=True,
41
+ ),
42
+ ]
32
43
 
33
44
 
34
45
  class MilvusAutoIndexTypedDict(CommonTypedDict, MilvusTypedDict): ...
@@ -45,7 +56,8 @@ def MilvusAutoIndex(**parameters: Unpack[MilvusAutoIndexTypedDict]):
45
56
  db_label=parameters["db_label"],
46
57
  uri=SecretStr(parameters["uri"]),
47
58
  user=parameters["user_name"],
48
- password=SecretStr(parameters["password"]),
59
+ password=SecretStr(parameters["password"]) if parameters["password"] else None,
60
+ num_shards=int(parameters["num_shards"]),
49
61
  ),
50
62
  db_case_config=AutoIndexConfig(),
51
63
  **parameters,
@@ -63,7 +75,8 @@ def MilvusFlat(**parameters: Unpack[MilvusAutoIndexTypedDict]):
63
75
  db_label=parameters["db_label"],
64
76
  uri=SecretStr(parameters["uri"]),
65
77
  user=parameters["user_name"],
66
- password=SecretStr(parameters["password"]),
78
+ password=SecretStr(parameters["password"]) if parameters["password"] else None,
79
+ num_shards=int(parameters["num_shards"]),
67
80
  ),
68
81
  db_case_config=FLATConfig(),
69
82
  **parameters,
@@ -85,6 +98,7 @@ def MilvusHNSW(**parameters: Unpack[MilvusHNSWTypedDict]):
85
98
  uri=SecretStr(parameters["uri"]),
86
99
  user=parameters["user_name"],
87
100
  password=SecretStr(parameters["password"]) if parameters["password"] else None,
101
+ num_shards=int(parameters["num_shards"]),
88
102
  ),
89
103
  db_case_config=HNSWConfig(
90
104
  M=parameters["m"],
@@ -109,7 +123,8 @@ def MilvusIVFFlat(**parameters: Unpack[MilvusIVFFlatTypedDict]):
109
123
  db_label=parameters["db_label"],
110
124
  uri=SecretStr(parameters["uri"]),
111
125
  user=parameters["user_name"],
112
- password=SecretStr(parameters["password"]),
126
+ password=SecretStr(parameters["password"]) if parameters["password"] else None,
127
+ num_shards=int(parameters["num_shards"]),
113
128
  ),
114
129
  db_case_config=IVFFlatConfig(
115
130
  nlist=parameters["nlist"],
@@ -130,7 +145,8 @@ def MilvusIVFSQ8(**parameters: Unpack[MilvusIVFFlatTypedDict]):
130
145
  db_label=parameters["db_label"],
131
146
  uri=SecretStr(parameters["uri"]),
132
147
  user=parameters["user_name"],
133
- password=SecretStr(parameters["password"]),
148
+ password=SecretStr(parameters["password"]) if parameters["password"] else None,
149
+ num_shards=int(parameters["num_shards"]),
134
150
  ),
135
151
  db_case_config=IVFSQ8Config(
136
152
  nlist=parameters["nlist"],
@@ -155,7 +171,8 @@ def MilvusDISKANN(**parameters: Unpack[MilvusDISKANNTypedDict]):
155
171
  db_label=parameters["db_label"],
156
172
  uri=SecretStr(parameters["uri"]),
157
173
  user=parameters["user_name"],
158
- password=SecretStr(parameters["password"]),
174
+ password=SecretStr(parameters["password"]) if parameters["password"] else None,
175
+ num_shards=int(parameters["num_shards"]),
159
176
  ),
160
177
  db_case_config=DISKANNConfig(
161
178
  search_list=parameters["search_list"],
@@ -183,7 +200,8 @@ def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]):
183
200
  db_label=parameters["db_label"],
184
201
  uri=SecretStr(parameters["uri"]),
185
202
  user=parameters["user_name"],
186
- password=SecretStr(parameters["password"]),
203
+ password=SecretStr(parameters["password"]) if parameters["password"] else None,
204
+ num_shards=int(parameters["num_shards"]),
187
205
  ),
188
206
  db_case_config=GPUIVFFlatConfig(
189
207
  nlist=parameters["nlist"],
@@ -217,7 +235,8 @@ def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]):
217
235
  db_label=parameters["db_label"],
218
236
  uri=SecretStr(parameters["uri"]),
219
237
  user=parameters["user_name"],
220
- password=SecretStr(parameters["password"]),
238
+ password=SecretStr(parameters["password"]) if parameters["password"] else None,
239
+ num_shards=int(parameters["num_shards"]),
221
240
  ),
222
241
  db_case_config=GPUBruteForceConfig(
223
242
  metric_type=parameters["metric_type"],
@@ -248,7 +267,8 @@ def MilvusGPUIVFPQ(**parameters: Unpack[MilvusGPUIVFPQTypedDict]):
248
267
  db_label=parameters["db_label"],
249
268
  uri=SecretStr(parameters["uri"]),
250
269
  user=parameters["user_name"],
251
- password=SecretStr(parameters["password"]),
270
+ password=SecretStr(parameters["password"]) if parameters["password"] else None,
271
+ num_shards=int(parameters["num_shards"]),
252
272
  ),
253
273
  db_case_config=GPUIVFPQConfig(
254
274
  nlist=parameters["nlist"],
@@ -287,7 +307,8 @@ def MilvusGPUCAGRA(**parameters: Unpack[MilvusGPUCAGRATypedDict]):
287
307
  db_label=parameters["db_label"],
288
308
  uri=SecretStr(parameters["uri"]),
289
309
  user=parameters["user_name"],
290
- password=SecretStr(parameters["password"]),
310
+ password=SecretStr(parameters["password"]) if parameters["password"] else None,
311
+ num_shards=int(parameters["num_shards"]),
291
312
  ),
292
313
  db_case_config=GPUCAGRAConfig(
293
314
  intermediate_graph_degree=parameters["intermediate_graph_degree"],
@@ -7,12 +7,14 @@ class MilvusConfig(DBConfig):
7
7
  uri: SecretStr = "http://localhost:19530"
8
8
  user: str | None = None
9
9
  password: SecretStr | None = None
10
+ num_shards: int = 1
10
11
 
11
12
  def to_dict(self) -> dict:
12
13
  return {
13
14
  "uri": self.uri.get_secret_value(),
14
15
  "user": self.user if self.user else None,
15
16
  "password": self.password.get_secret_value() if self.password else None,
17
+ "num_shards": self.num_shards,
16
18
  }
17
19
 
18
20
  @validator("*")
@@ -40,7 +40,12 @@ class Milvus(VectorDB):
40
40
 
41
41
  from pymilvus import connections
42
42
 
43
- connections.connect(**self.db_config, timeout=30)
43
+ connections.connect(
44
+ uri=self.db_config.get("uri"),
45
+ user=self.db_config.get("user"),
46
+ password=self.db_config.get("password"),
47
+ timeout=30,
48
+ )
44
49
  if drop_old and utility.has_collection(self.collection_name):
45
50
  log.info(f"{self.name} client drop_old collection: {self.collection_name}")
46
51
  utility.drop_collection(self.collection_name)
@@ -59,6 +64,7 @@ class Milvus(VectorDB):
59
64
  name=self.collection_name,
60
65
  schema=CollectionSchema(fields),
61
66
  consistency_level="Session",
67
+ num_shards=self.db_config.get("num_shards"),
62
68
  )
63
69
 
64
70
  log.info(f"{self.name} create index: index_params: {self.case_config.index_param()}")
@@ -0,0 +1,60 @@
1
+ from typing import Annotated, Unpack
2
+
3
+ import click
4
+ from pydantic import SecretStr
5
+
6
+ from vectordb_bench.backend.clients import DB
7
+ from vectordb_bench.cli.cli import (
8
+ CommonTypedDict,
9
+ cli,
10
+ click_parameter_decorators_from_typed_dict,
11
+ run,
12
+ )
13
+
14
+ DBTYPE = DB.QdrantLocal
15
+
16
+
17
+ class QdrantLocalTypedDict(CommonTypedDict):
18
+ url: Annotated[
19
+ str,
20
+ click.option("--url", type=str, help="Qdrant url", required=True),
21
+ ]
22
+ on_disk: Annotated[
23
+ bool,
24
+ click.option("--on-disk", type=bool, default=False, help="Store the vectors and the HNSW index on disk"),
25
+ ]
26
+ m: Annotated[
27
+ int,
28
+ click.option("--m", type=int, default=16, help="HNSW index parameter m, set 0 to disable the index"),
29
+ ]
30
+ ef_construct: Annotated[
31
+ int,
32
+ click.option("--ef-construct", type=int, default=200, help="HNSW index parameter ef_construct"),
33
+ ]
34
+ hnsw_ef: Annotated[
35
+ int,
36
+ click.option(
37
+ "--hnsw-ef",
38
+ type=int,
39
+ default=0,
40
+ help="HNSW index parameter hnsw_ef, set 0 to use ef_construct for search",
41
+ ),
42
+ ]
43
+
44
+
45
+ @cli.command()
46
+ @click_parameter_decorators_from_typed_dict(QdrantLocalTypedDict)
47
+ def QdrantLocal(**parameters: Unpack[QdrantLocalTypedDict]):
48
+ from .config import QdrantLocalConfig, QdrantLocalIndexConfig
49
+
50
+ run(
51
+ db=DBTYPE,
52
+ db_config=QdrantLocalConfig(url=SecretStr(parameters["url"])),
53
+ db_case_config=QdrantLocalIndexConfig(
54
+ on_disk=parameters["on_disk"],
55
+ m=parameters["m"],
56
+ ef_construct=parameters["ef_construct"],
57
+ hnsw_ef=parameters["hnsw_ef"],
58
+ ),
59
+ **parameters,
60
+ )
@@ -0,0 +1,47 @@
1
+ from pydantic import BaseModel, SecretStr
2
+
3
+ from ..api import DBCaseConfig, DBConfig, MetricType
4
+
5
+
6
+ class QdrantLocalConfig(DBConfig):
7
+ url: SecretStr
8
+
9
+ def to_dict(self) -> dict:
10
+ return {
11
+ "url": self.url.get_secret_value(),
12
+ }
13
+
14
+
15
+ class QdrantLocalIndexConfig(BaseModel, DBCaseConfig):
16
+ metric_type: MetricType | None = None
17
+ m: int
18
+ ef_construct: int
19
+ hnsw_ef: int | None = 0
20
+ on_disk: bool | None = False
21
+
22
+ def parse_metric(self) -> str:
23
+ if self.metric_type == MetricType.L2:
24
+ return "Euclid"
25
+
26
+ if self.metric_type == MetricType.IP:
27
+ return "Dot"
28
+
29
+ return "Cosine"
30
+
31
+ def index_param(self) -> dict:
32
+ return {
33
+ "distance": self.parse_metric(),
34
+ "m": self.m,
35
+ "ef_construct": self.ef_construct,
36
+ "on_disk": self.on_disk,
37
+ }
38
+
39
+ def search_param(self) -> dict:
40
+ search_params = {
41
+ "exact": False, # Force to use ANNs
42
+ }
43
+
44
+ if self.hnsw_ef != 0:
45
+ search_params["hnsw_ef"] = self.hnsw_ef
46
+
47
+ return search_params
@@ -0,0 +1,232 @@
1
+ """Wrapper around the Qdrant over VectorDB"""
2
+
3
+ import logging
4
+ import time
5
+ from collections.abc import Iterable
6
+ from contextlib import contextmanager
7
+
8
+ from qdrant_client import QdrantClient
9
+ from qdrant_client.http.models import (
10
+ Batch,
11
+ CollectionStatus,
12
+ FieldCondition,
13
+ Filter,
14
+ HnswConfigDiff,
15
+ OptimizersConfigDiff,
16
+ PayloadSchemaType,
17
+ Range,
18
+ SearchParams,
19
+ VectorParams,
20
+ )
21
+
22
+ from ..api import VectorDB
23
+ from .config import QdrantLocalIndexConfig
24
+
25
+ log = logging.getLogger(__name__)
26
+
27
+ SECONDS_WAITING_FOR_INDEXING_API_CALL = 5
28
+ QDRANT_BATCH_SIZE = 100
29
+
30
+
31
+ def qdrant_collection_exists(client: QdrantClient, collection_name: str) -> bool:
32
+ collection_exists = True
33
+
34
+ try:
35
+ client.get_collection(collection_name)
36
+ except Exception:
37
+ collection_exists = False
38
+
39
+ return collection_exists
40
+
41
+
42
+ class QdrantLocal(VectorDB):
43
+ def __init__(
44
+ self,
45
+ dim: int,
46
+ db_config: dict,
47
+ db_case_config: QdrantLocalIndexConfig,
48
+ collection_name: str = "QdrantLocalCollection",
49
+ drop_old: bool = False,
50
+ name: str = "QdrantLocal",
51
+ **kwargs,
52
+ ):
53
+ """Initialize wrapper around the qdrant."""
54
+ self.name = name
55
+ self.db_config = db_config
56
+ self.case_config = db_case_config
57
+ self.search_parameter = self.case_config.search_param()
58
+ self.collection_name = collection_name
59
+ self.client = None
60
+
61
+ self._primary_field = "pk"
62
+ self._vector_field = "vector"
63
+
64
+ client = QdrantClient(**self.db_config)
65
+
66
+ # Lets just print the parameters here for double check
67
+ log.info(f"Case config: {self.case_config.index_param()}")
68
+ log.info(f"Search parameter: {self.search_parameter}")
69
+
70
+ if drop_old and qdrant_collection_exists(client, self.collection_name):
71
+ log.info(f"{self.name} client drop_old collection: {self.collection_name}")
72
+ client.delete_collection(self.collection_name)
73
+
74
+ if not qdrant_collection_exists(client, self.collection_name):
75
+ log.info(f"{self.name} create collection: {self.collection_name}")
76
+ self._create_collection(dim, client)
77
+
78
+ client = None
79
+
80
+ @contextmanager
81
+ def init(self):
82
+ """
83
+ Examples:
84
+ >>> with self.init():
85
+ >>> self.insert_embeddings()
86
+ >>> self.search_embedding()
87
+ """
88
+ # create connection
89
+ self.client = QdrantClient(**self.db_config)
90
+ yield
91
+ self.client = None
92
+ del self.client
93
+
94
+ def _create_collection(self, dim: int, qdrant_client: QdrantClient):
95
+ log.info(f"Create collection: {self.collection_name}")
96
+ log.info(
97
+ f"Index parameters: m={self.case_config.index_param()['m']}, "
98
+ f"ef_construct={self.case_config.index_param()['ef_construct']}, "
99
+ f"on_disk={self.case_config.index_param()['on_disk']}"
100
+ )
101
+
102
+ # If the on_disk is true, we enable both on disk index and vectors.
103
+ try:
104
+ qdrant_client.create_collection(
105
+ collection_name=self.collection_name,
106
+ vectors_config=VectorParams(
107
+ size=dim,
108
+ distance=self.case_config.index_param()["distance"],
109
+ on_disk=self.case_config.index_param()["on_disk"],
110
+ ),
111
+ hnsw_config=HnswConfigDiff(
112
+ m=self.case_config.index_param()["m"],
113
+ ef_construct=self.case_config.index_param()["ef_construct"],
114
+ on_disk=self.case_config.index_param()["on_disk"],
115
+ ),
116
+ )
117
+
118
+ qdrant_client.create_payload_index(
119
+ collection_name=self.collection_name,
120
+ field_name=self._primary_field,
121
+ field_schema=PayloadSchemaType.INTEGER,
122
+ )
123
+
124
+ except Exception as e:
125
+ if "already exists!" in str(e):
126
+ return
127
+ log.warning(f"Failed to create collection: {self.collection_name} error: {e}")
128
+ raise e from None
129
+
130
+ def optimize(self, data_size: int | None = None):
131
+ assert self.client, "Please call self.init() before"
132
+ # wait for vectors to be fully indexed
133
+ try:
134
+ while True:
135
+ info = self.client.get_collection(self.collection_name)
136
+ time.sleep(SECONDS_WAITING_FOR_INDEXING_API_CALL)
137
+ if info.status != CollectionStatus.GREEN:
138
+ continue
139
+ if info.status == CollectionStatus.GREEN:
140
+ log.info(f"Finishing building index for collection: {self.collection_name}")
141
+ msg = (
142
+ f"Stored vectors: {info.vectors_count}, Indexed vectors: {info.indexed_vectors_count}, "
143
+ f"Collection status: {info.indexed_vectors_count}"
144
+ )
145
+ log.info(msg)
146
+ return
147
+
148
+ except Exception as e:
149
+ log.warning(f"QdrantCloud ready to search error: {e}")
150
+ raise e from None
151
+
152
+ def insert_embeddings(
153
+ self,
154
+ embeddings: Iterable[list[float]],
155
+ metadata: list[int],
156
+ **kwargs,
157
+ ) -> tuple[int, Exception]:
158
+ """Insert embeddings into the database.
159
+
160
+ Args:
161
+ embeddings(list[list[float]]): list of embeddings
162
+ metadata(list[int]): list of metadata
163
+ kwargs: other arguments
164
+
165
+ Returns:
166
+ tuple[int, Exception]: number of embeddings inserted and exception if any
167
+ """
168
+ assert self.client is not None
169
+ assert len(embeddings) == len(metadata)
170
+ insert_count = 0
171
+
172
+ # disable indexing for quick insertion
173
+ self.client.update_collection(
174
+ collection_name=self.collection_name,
175
+ optimizer_config=OptimizersConfigDiff(indexing_threshold=0),
176
+ )
177
+ try:
178
+ for offset in range(0, len(embeddings), QDRANT_BATCH_SIZE):
179
+ vectors = embeddings[offset : offset + QDRANT_BATCH_SIZE]
180
+ ids = metadata[offset : offset + QDRANT_BATCH_SIZE]
181
+ payloads = [{self._primary_field: v} for v in ids]
182
+ _ = self.client.upsert(
183
+ collection_name=self.collection_name,
184
+ wait=True,
185
+ points=Batch(ids=ids, payloads=payloads, vectors=vectors),
186
+ )
187
+ insert_count += QDRANT_BATCH_SIZE
188
+ # enable indexing after insertion
189
+ self.client.update_collection(
190
+ collection_name=self.collection_name,
191
+ optimizer_config=OptimizersConfigDiff(indexing_threshold=100),
192
+ )
193
+
194
+ except Exception as e:
195
+ log.info(f"Failed to insert data, {e}")
196
+ return insert_count, e
197
+ else:
198
+ return insert_count, None
199
+
200
+ def search_embedding(
201
+ self,
202
+ query: list[float],
203
+ k: int = 100,
204
+ filters: dict | None = None,
205
+ timeout: int | None = None,
206
+ ) -> list[int]:
207
+ """Perform a search on a query embedding and return results with score.
208
+ Should call self.init() first.
209
+ """
210
+ assert self.client is not None
211
+
212
+ f = None
213
+ if filters:
214
+ f = Filter(
215
+ must=[
216
+ FieldCondition(
217
+ key=self._primary_field,
218
+ range=Range(
219
+ gt=filters.get("id"),
220
+ ),
221
+ ),
222
+ ],
223
+ )
224
+ res = self.client.query_points(
225
+ collection_name=self.collection_name,
226
+ query=query,
227
+ limit=k,
228
+ query_filter=f,
229
+ search_params=SearchParams(**self.search_parameter),
230
+ ).points
231
+
232
+ return [result.id for result in res]
@@ -15,12 +15,33 @@ from .. import DB
15
15
  class WeaviateTypedDict(CommonTypedDict):
16
16
  api_key: Annotated[
17
17
  str,
18
- click.option("--api-key", type=str, help="Weaviate api key", required=True),
18
+ click.option("--api-key", type=str, help="Weaviate api key", required=False, default=""),
19
19
  ]
20
20
  url: Annotated[
21
21
  str,
22
22
  click.option("--url", type=str, help="Weaviate url", required=True),
23
23
  ]
24
+ no_auth: Annotated[
25
+ bool,
26
+ click.option(
27
+ "--no-auth",
28
+ is_flag=True,
29
+ help="Do not use api-key, set it to true if you are using a local setup. Default is False.",
30
+ default=False,
31
+ ),
32
+ ]
33
+ m: Annotated[
34
+ int,
35
+ click.option("--m", type=int, default=16, help="HNSW index parameter m."),
36
+ ]
37
+ ef_construct: Annotated[
38
+ int,
39
+ click.option("--ef-construction", type=int, default=256, help="HNSW index parameter ef_construction"),
40
+ ]
41
+ ef: Annotated[
42
+ int,
43
+ click.option("--ef", type=int, default=256, help="HNSW index parameter ef for search"),
44
+ ]
24
45
 
25
46
 
26
47
  @cli.command()
@@ -32,9 +53,14 @@ def Weaviate(**parameters: Unpack[WeaviateTypedDict]):
32
53
  db=DB.WeaviateCloud,
33
54
  db_config=WeaviateConfig(
34
55
  db_label=parameters["db_label"],
35
- api_key=SecretStr(parameters["api_key"]),
56
+ api_key=SecretStr(parameters["api_key"]) if parameters["api_key"] != "" else SecretStr("-"),
36
57
  url=SecretStr(parameters["url"]),
58
+ no_auth=parameters["no_auth"],
59
+ ),
60
+ db_case_config=WeaviateIndexConfig(
61
+ efConstruction=parameters["ef_construction"],
62
+ maxConnections=parameters["m"],
63
+ ef=parameters["ef"],
37
64
  ),
38
- db_case_config=WeaviateIndexConfig(ef=256, efConstruction=256, maxConnections=16),
39
65
  **parameters,
40
66
  )
@@ -6,11 +6,13 @@ from ..api import DBCaseConfig, DBConfig, MetricType
6
6
  class WeaviateConfig(DBConfig):
7
7
  url: SecretStr
8
8
  api_key: SecretStr
9
+ no_auth: bool | None = False
9
10
 
10
11
  def to_dict(self) -> dict:
11
12
  return {
12
13
  "url": self.url.get_secret_value(),
13
14
  "auth_client_secret": self.api_key.get_secret_value(),
15
+ "no_auth": self.no_auth,
14
16
  }
15
17
 
16
18
 
@@ -38,6 +38,11 @@ class WeaviateCloud(VectorDB):
38
38
  self._vector_field = "vector"
39
39
  self._index_name = "vector_idx"
40
40
 
41
+ # If local setup is used, we
42
+ if db_config["no_auth"]:
43
+ del db_config["auth_client_secret"]
44
+ del db_config["no_auth"]
45
+
41
46
  from weaviate import Client
42
47
 
43
48
  client = Client(**db_config)