vectordb-bench 0.0.29__py3-none-any.whl → 0.0.30__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vectordb_bench/backend/clients/__init__.py +16 -0
- vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +180 -15
- vectordb_bench/backend/clients/aws_opensearch/cli.py +51 -21
- vectordb_bench/backend/clients/aws_opensearch/config.py +37 -14
- vectordb_bench/backend/clients/lancedb/cli.py +62 -8
- vectordb_bench/backend/clients/lancedb/config.py +14 -1
- vectordb_bench/backend/clients/lancedb/lancedb.py +21 -9
- vectordb_bench/backend/clients/memorydb/memorydb.py +2 -2
- vectordb_bench/backend/clients/milvus/cli.py +30 -9
- vectordb_bench/backend/clients/milvus/config.py +2 -0
- vectordb_bench/backend/clients/milvus/milvus.py +7 -1
- vectordb_bench/backend/clients/qdrant_local/cli.py +60 -0
- vectordb_bench/backend/clients/qdrant_local/config.py +47 -0
- vectordb_bench/backend/clients/qdrant_local/qdrant_local.py +232 -0
- vectordb_bench/backend/clients/weaviate_cloud/cli.py +29 -3
- vectordb_bench/backend/clients/weaviate_cloud/config.py +2 -0
- vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +5 -0
- vectordb_bench/cli/batch_cli.py +121 -0
- vectordb_bench/cli/vectordbbench.py +4 -0
- vectordb_bench/config-files/batch_sample_config.yml +17 -0
- vectordb_bench/frontend/config/dbCaseConfigs.py +113 -1
- vectordb_bench/models.py +7 -0
- {vectordb_bench-0.0.29.dist-info → vectordb_bench-0.0.30.dist-info}/METADATA +48 -2
- {vectordb_bench-0.0.29.dist-info → vectordb_bench-0.0.30.dist-info}/RECORD +28 -23
- {vectordb_bench-0.0.29.dist-info → vectordb_bench-0.0.30.dist-info}/WHEEL +1 -1
- {vectordb_bench-0.0.29.dist-info → vectordb_bench-0.0.30.dist-info}/entry_points.txt +0 -0
- {vectordb_bench-0.0.29.dist-info → vectordb_bench-0.0.30.dist-info}/licenses/LICENSE +0 -0
- {vectordb_bench-0.0.29.dist-info → vectordb_bench-0.0.30.dist-info}/top_level.txt +0 -0
@@ -25,6 +25,7 @@ class LanceDBIndexConfig(BaseModel, DBCaseConfig):
|
|
25
25
|
nbits: int = 8 # Must be 4 or 8
|
26
26
|
sample_rate: int = 256
|
27
27
|
max_iterations: int = 50
|
28
|
+
nprobes: int = 0
|
28
29
|
|
29
30
|
def index_param(self) -> dict:
|
30
31
|
if self.index not in [
|
@@ -52,7 +53,11 @@ class LanceDBIndexConfig(BaseModel, DBCaseConfig):
|
|
52
53
|
return params
|
53
54
|
|
54
55
|
def search_param(self) -> dict:
|
55
|
-
|
56
|
+
params = {}
|
57
|
+
if self.nprobes > 0:
|
58
|
+
params["nprobes"] = self.nprobes
|
59
|
+
|
60
|
+
return params
|
56
61
|
|
57
62
|
def parse_metric(self) -> str:
|
58
63
|
if self.metric_type in [MetricType.L2, MetricType.COSINE]:
|
@@ -81,6 +86,7 @@ class LanceDBHNSWIndexConfig(LanceDBIndexConfig):
|
|
81
86
|
index: IndexType = IndexType.HNSW
|
82
87
|
m: int = 0
|
83
88
|
ef_construction: int = 0
|
89
|
+
ef: int = 0
|
84
90
|
|
85
91
|
def index_param(self) -> dict:
|
86
92
|
params = LanceDBIndexConfig.index_param(self)
|
@@ -94,6 +100,13 @@ class LanceDBHNSWIndexConfig(LanceDBIndexConfig):
|
|
94
100
|
|
95
101
|
return params
|
96
102
|
|
103
|
+
def search_param(self) -> dict:
|
104
|
+
params = {}
|
105
|
+
if self.ef != 0:
|
106
|
+
params = {"ef": self.ef}
|
107
|
+
|
108
|
+
return params
|
109
|
+
|
97
110
|
|
98
111
|
_lancedb_case_config = {
|
99
112
|
IndexType.IVFPQ: LanceDBIndexConfig,
|
@@ -32,6 +32,10 @@ class LanceDB(VectorDB):
|
|
32
32
|
self.table_name = collection_name
|
33
33
|
self.dim = dim
|
34
34
|
self.uri = db_config["uri"]
|
35
|
+
# avoid the search_param being called every time during the search process
|
36
|
+
self.search_config = db_case_config.search_param()
|
37
|
+
|
38
|
+
log.info(f"Search config: {self.search_config}")
|
35
39
|
|
36
40
|
db = lancedb.connect(self.uri)
|
37
41
|
|
@@ -45,7 +49,7 @@ class LanceDB(VectorDB):
|
|
45
49
|
db.open_table(self.table_name)
|
46
50
|
except Exception:
|
47
51
|
schema = pa.schema(
|
48
|
-
[pa.field("id", pa.int64()), pa.field("vector", pa.list_(pa.
|
52
|
+
[pa.field("id", pa.int64()), pa.field("vector", pa.list_(pa.float32(), list_size=self.dim))]
|
49
53
|
)
|
50
54
|
db.create_table(self.table_name, schema=schema, mode="overwrite")
|
51
55
|
|
@@ -77,20 +81,28 @@ class LanceDB(VectorDB):
|
|
77
81
|
filters: dict | None = None,
|
78
82
|
) -> list[int]:
|
79
83
|
if filters:
|
80
|
-
results = (
|
81
|
-
|
82
|
-
.
|
83
|
-
|
84
|
-
.
|
85
|
-
|
86
|
-
|
84
|
+
results = self.table.search(query).select(["id"]).where(f"id >= {filters['id']}", prefilter=True).limit(k)
|
85
|
+
if self.case_config.index == IndexType.IVFPQ and "nprobes" in self.search_config:
|
86
|
+
results = results.nprobes(self.search_config["nprobes"]).to_list()
|
87
|
+
elif self.case_config.index == IndexType.HNSW and "ef" in self.search_config:
|
88
|
+
results = results.ef(self.search_config["ef"]).to_list()
|
89
|
+
else:
|
90
|
+
results = results.to_list()
|
87
91
|
else:
|
88
|
-
results = self.table.search(query).select(["id"]).limit(k)
|
92
|
+
results = self.table.search(query).select(["id"]).limit(k)
|
93
|
+
if self.case_config.index == IndexType.IVFPQ and "nprobes" in self.search_config:
|
94
|
+
results = results.nprobes(self.search_config["nprobes"]).to_list()
|
95
|
+
elif self.case_config.index == IndexType.HNSW and "ef" in self.search_config:
|
96
|
+
results = results.ef(self.search_config["ef"]).to_list()
|
97
|
+
else:
|
98
|
+
results = results.to_list()
|
99
|
+
|
89
100
|
return [int(result["id"]) for result in results]
|
90
101
|
|
91
102
|
def optimize(self, data_size: int | None = None):
|
92
103
|
if self.table and hasattr(self, "case_config") and self.case_config.index != IndexType.NONE:
|
93
104
|
log.info(f"Creating index for LanceDB table ({self.table_name})")
|
105
|
+
log.info(f"Index parameters: {self.case_config.index_param()}")
|
94
106
|
self.table.create_index(**self.case_config.index_param())
|
95
107
|
# Better recall with IVF_PQ (though still bad) but breaks HNSW: https://github.com/lancedb/lancedb/issues/2369
|
96
108
|
if self.case_config.index in (IndexType.IVFPQ, IndexType.AUTOINDEX):
|
@@ -9,10 +9,10 @@ import redis
|
|
9
9
|
from redis import Redis
|
10
10
|
from redis.cluster import RedisCluster
|
11
11
|
from redis.commands.search.field import NumericField, TagField, VectorField
|
12
|
-
from redis.commands.search.indexDefinition import IndexDefinition
|
12
|
+
from redis.commands.search.indexDefinition import IndexDefinition, IndexType
|
13
13
|
from redis.commands.search.query import Query
|
14
14
|
|
15
|
-
from ..api import
|
15
|
+
from ..api import VectorDB
|
16
16
|
from .config import MemoryDBIndexConfig
|
17
17
|
|
18
18
|
log = logging.getLogger(__name__)
|
@@ -29,6 +29,17 @@ class MilvusTypedDict(TypedDict):
|
|
29
29
|
str | None,
|
30
30
|
click.option("--password", type=str, help="Db password", required=False),
|
31
31
|
]
|
32
|
+
num_shards: Annotated[
|
33
|
+
int,
|
34
|
+
click.option(
|
35
|
+
"--num-shards",
|
36
|
+
type=int,
|
37
|
+
help="Number of shards",
|
38
|
+
required=False,
|
39
|
+
default=1,
|
40
|
+
show_default=True,
|
41
|
+
),
|
42
|
+
]
|
32
43
|
|
33
44
|
|
34
45
|
class MilvusAutoIndexTypedDict(CommonTypedDict, MilvusTypedDict): ...
|
@@ -45,7 +56,8 @@ def MilvusAutoIndex(**parameters: Unpack[MilvusAutoIndexTypedDict]):
|
|
45
56
|
db_label=parameters["db_label"],
|
46
57
|
uri=SecretStr(parameters["uri"]),
|
47
58
|
user=parameters["user_name"],
|
48
|
-
password=SecretStr(parameters["password"]),
|
59
|
+
password=SecretStr(parameters["password"]) if parameters["password"] else None,
|
60
|
+
num_shards=int(parameters["num_shards"]),
|
49
61
|
),
|
50
62
|
db_case_config=AutoIndexConfig(),
|
51
63
|
**parameters,
|
@@ -63,7 +75,8 @@ def MilvusFlat(**parameters: Unpack[MilvusAutoIndexTypedDict]):
|
|
63
75
|
db_label=parameters["db_label"],
|
64
76
|
uri=SecretStr(parameters["uri"]),
|
65
77
|
user=parameters["user_name"],
|
66
|
-
password=SecretStr(parameters["password"]),
|
78
|
+
password=SecretStr(parameters["password"]) if parameters["password"] else None,
|
79
|
+
num_shards=int(parameters["num_shards"]),
|
67
80
|
),
|
68
81
|
db_case_config=FLATConfig(),
|
69
82
|
**parameters,
|
@@ -85,6 +98,7 @@ def MilvusHNSW(**parameters: Unpack[MilvusHNSWTypedDict]):
|
|
85
98
|
uri=SecretStr(parameters["uri"]),
|
86
99
|
user=parameters["user_name"],
|
87
100
|
password=SecretStr(parameters["password"]) if parameters["password"] else None,
|
101
|
+
num_shards=int(parameters["num_shards"]),
|
88
102
|
),
|
89
103
|
db_case_config=HNSWConfig(
|
90
104
|
M=parameters["m"],
|
@@ -109,7 +123,8 @@ def MilvusIVFFlat(**parameters: Unpack[MilvusIVFFlatTypedDict]):
|
|
109
123
|
db_label=parameters["db_label"],
|
110
124
|
uri=SecretStr(parameters["uri"]),
|
111
125
|
user=parameters["user_name"],
|
112
|
-
password=SecretStr(parameters["password"]),
|
126
|
+
password=SecretStr(parameters["password"]) if parameters["password"] else None,
|
127
|
+
num_shards=int(parameters["num_shards"]),
|
113
128
|
),
|
114
129
|
db_case_config=IVFFlatConfig(
|
115
130
|
nlist=parameters["nlist"],
|
@@ -130,7 +145,8 @@ def MilvusIVFSQ8(**parameters: Unpack[MilvusIVFFlatTypedDict]):
|
|
130
145
|
db_label=parameters["db_label"],
|
131
146
|
uri=SecretStr(parameters["uri"]),
|
132
147
|
user=parameters["user_name"],
|
133
|
-
password=SecretStr(parameters["password"]),
|
148
|
+
password=SecretStr(parameters["password"]) if parameters["password"] else None,
|
149
|
+
num_shards=int(parameters["num_shards"]),
|
134
150
|
),
|
135
151
|
db_case_config=IVFSQ8Config(
|
136
152
|
nlist=parameters["nlist"],
|
@@ -155,7 +171,8 @@ def MilvusDISKANN(**parameters: Unpack[MilvusDISKANNTypedDict]):
|
|
155
171
|
db_label=parameters["db_label"],
|
156
172
|
uri=SecretStr(parameters["uri"]),
|
157
173
|
user=parameters["user_name"],
|
158
|
-
password=SecretStr(parameters["password"]),
|
174
|
+
password=SecretStr(parameters["password"]) if parameters["password"] else None,
|
175
|
+
num_shards=int(parameters["num_shards"]),
|
159
176
|
),
|
160
177
|
db_case_config=DISKANNConfig(
|
161
178
|
search_list=parameters["search_list"],
|
@@ -183,7 +200,8 @@ def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]):
|
|
183
200
|
db_label=parameters["db_label"],
|
184
201
|
uri=SecretStr(parameters["uri"]),
|
185
202
|
user=parameters["user_name"],
|
186
|
-
password=SecretStr(parameters["password"]),
|
203
|
+
password=SecretStr(parameters["password"]) if parameters["password"] else None,
|
204
|
+
num_shards=int(parameters["num_shards"]),
|
187
205
|
),
|
188
206
|
db_case_config=GPUIVFFlatConfig(
|
189
207
|
nlist=parameters["nlist"],
|
@@ -217,7 +235,8 @@ def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]):
|
|
217
235
|
db_label=parameters["db_label"],
|
218
236
|
uri=SecretStr(parameters["uri"]),
|
219
237
|
user=parameters["user_name"],
|
220
|
-
password=SecretStr(parameters["password"]),
|
238
|
+
password=SecretStr(parameters["password"]) if parameters["password"] else None,
|
239
|
+
num_shards=int(parameters["num_shards"]),
|
221
240
|
),
|
222
241
|
db_case_config=GPUBruteForceConfig(
|
223
242
|
metric_type=parameters["metric_type"],
|
@@ -248,7 +267,8 @@ def MilvusGPUIVFPQ(**parameters: Unpack[MilvusGPUIVFPQTypedDict]):
|
|
248
267
|
db_label=parameters["db_label"],
|
249
268
|
uri=SecretStr(parameters["uri"]),
|
250
269
|
user=parameters["user_name"],
|
251
|
-
password=SecretStr(parameters["password"]),
|
270
|
+
password=SecretStr(parameters["password"]) if parameters["password"] else None,
|
271
|
+
num_shards=int(parameters["num_shards"]),
|
252
272
|
),
|
253
273
|
db_case_config=GPUIVFPQConfig(
|
254
274
|
nlist=parameters["nlist"],
|
@@ -287,7 +307,8 @@ def MilvusGPUCAGRA(**parameters: Unpack[MilvusGPUCAGRATypedDict]):
|
|
287
307
|
db_label=parameters["db_label"],
|
288
308
|
uri=SecretStr(parameters["uri"]),
|
289
309
|
user=parameters["user_name"],
|
290
|
-
password=SecretStr(parameters["password"]),
|
310
|
+
password=SecretStr(parameters["password"]) if parameters["password"] else None,
|
311
|
+
num_shards=int(parameters["num_shards"]),
|
291
312
|
),
|
292
313
|
db_case_config=GPUCAGRAConfig(
|
293
314
|
intermediate_graph_degree=parameters["intermediate_graph_degree"],
|
@@ -7,12 +7,14 @@ class MilvusConfig(DBConfig):
|
|
7
7
|
uri: SecretStr = "http://localhost:19530"
|
8
8
|
user: str | None = None
|
9
9
|
password: SecretStr | None = None
|
10
|
+
num_shards: int = 1
|
10
11
|
|
11
12
|
def to_dict(self) -> dict:
|
12
13
|
return {
|
13
14
|
"uri": self.uri.get_secret_value(),
|
14
15
|
"user": self.user if self.user else None,
|
15
16
|
"password": self.password.get_secret_value() if self.password else None,
|
17
|
+
"num_shards": self.num_shards,
|
16
18
|
}
|
17
19
|
|
18
20
|
@validator("*")
|
@@ -40,7 +40,12 @@ class Milvus(VectorDB):
|
|
40
40
|
|
41
41
|
from pymilvus import connections
|
42
42
|
|
43
|
-
connections.connect(
|
43
|
+
connections.connect(
|
44
|
+
uri=self.db_config.get("uri"),
|
45
|
+
user=self.db_config.get("user"),
|
46
|
+
password=self.db_config.get("password"),
|
47
|
+
timeout=30,
|
48
|
+
)
|
44
49
|
if drop_old and utility.has_collection(self.collection_name):
|
45
50
|
log.info(f"{self.name} client drop_old collection: {self.collection_name}")
|
46
51
|
utility.drop_collection(self.collection_name)
|
@@ -59,6 +64,7 @@ class Milvus(VectorDB):
|
|
59
64
|
name=self.collection_name,
|
60
65
|
schema=CollectionSchema(fields),
|
61
66
|
consistency_level="Session",
|
67
|
+
num_shards=self.db_config.get("num_shards"),
|
62
68
|
)
|
63
69
|
|
64
70
|
log.info(f"{self.name} create index: index_params: {self.case_config.index_param()}")
|
@@ -0,0 +1,60 @@
|
|
1
|
+
from typing import Annotated, Unpack
|
2
|
+
|
3
|
+
import click
|
4
|
+
from pydantic import SecretStr
|
5
|
+
|
6
|
+
from vectordb_bench.backend.clients import DB
|
7
|
+
from vectordb_bench.cli.cli import (
|
8
|
+
CommonTypedDict,
|
9
|
+
cli,
|
10
|
+
click_parameter_decorators_from_typed_dict,
|
11
|
+
run,
|
12
|
+
)
|
13
|
+
|
14
|
+
DBTYPE = DB.QdrantLocal
|
15
|
+
|
16
|
+
|
17
|
+
class QdrantLocalTypedDict(CommonTypedDict):
|
18
|
+
url: Annotated[
|
19
|
+
str,
|
20
|
+
click.option("--url", type=str, help="Qdrant url", required=True),
|
21
|
+
]
|
22
|
+
on_disk: Annotated[
|
23
|
+
bool,
|
24
|
+
click.option("--on-disk", type=bool, default=False, help="Store the vectors and the HNSW index on disk"),
|
25
|
+
]
|
26
|
+
m: Annotated[
|
27
|
+
int,
|
28
|
+
click.option("--m", type=int, default=16, help="HNSW index parameter m, set 0 to disable the index"),
|
29
|
+
]
|
30
|
+
ef_construct: Annotated[
|
31
|
+
int,
|
32
|
+
click.option("--ef-construct", type=int, default=200, help="HNSW index parameter ef_construct"),
|
33
|
+
]
|
34
|
+
hnsw_ef: Annotated[
|
35
|
+
int,
|
36
|
+
click.option(
|
37
|
+
"--hnsw-ef",
|
38
|
+
type=int,
|
39
|
+
default=0,
|
40
|
+
help="HNSW index parameter hnsw_ef, set 0 to use ef_construct for search",
|
41
|
+
),
|
42
|
+
]
|
43
|
+
|
44
|
+
|
45
|
+
@cli.command()
|
46
|
+
@click_parameter_decorators_from_typed_dict(QdrantLocalTypedDict)
|
47
|
+
def QdrantLocal(**parameters: Unpack[QdrantLocalTypedDict]):
|
48
|
+
from .config import QdrantLocalConfig, QdrantLocalIndexConfig
|
49
|
+
|
50
|
+
run(
|
51
|
+
db=DBTYPE,
|
52
|
+
db_config=QdrantLocalConfig(url=SecretStr(parameters["url"])),
|
53
|
+
db_case_config=QdrantLocalIndexConfig(
|
54
|
+
on_disk=parameters["on_disk"],
|
55
|
+
m=parameters["m"],
|
56
|
+
ef_construct=parameters["ef_construct"],
|
57
|
+
hnsw_ef=parameters["hnsw_ef"],
|
58
|
+
),
|
59
|
+
**parameters,
|
60
|
+
)
|
@@ -0,0 +1,47 @@
|
|
1
|
+
from pydantic import BaseModel, SecretStr
|
2
|
+
|
3
|
+
from ..api import DBCaseConfig, DBConfig, MetricType
|
4
|
+
|
5
|
+
|
6
|
+
class QdrantLocalConfig(DBConfig):
|
7
|
+
url: SecretStr
|
8
|
+
|
9
|
+
def to_dict(self) -> dict:
|
10
|
+
return {
|
11
|
+
"url": self.url.get_secret_value(),
|
12
|
+
}
|
13
|
+
|
14
|
+
|
15
|
+
class QdrantLocalIndexConfig(BaseModel, DBCaseConfig):
|
16
|
+
metric_type: MetricType | None = None
|
17
|
+
m: int
|
18
|
+
ef_construct: int
|
19
|
+
hnsw_ef: int | None = 0
|
20
|
+
on_disk: bool | None = False
|
21
|
+
|
22
|
+
def parse_metric(self) -> str:
|
23
|
+
if self.metric_type == MetricType.L2:
|
24
|
+
return "Euclid"
|
25
|
+
|
26
|
+
if self.metric_type == MetricType.IP:
|
27
|
+
return "Dot"
|
28
|
+
|
29
|
+
return "Cosine"
|
30
|
+
|
31
|
+
def index_param(self) -> dict:
|
32
|
+
return {
|
33
|
+
"distance": self.parse_metric(),
|
34
|
+
"m": self.m,
|
35
|
+
"ef_construct": self.ef_construct,
|
36
|
+
"on_disk": self.on_disk,
|
37
|
+
}
|
38
|
+
|
39
|
+
def search_param(self) -> dict:
|
40
|
+
search_params = {
|
41
|
+
"exact": False, # Force to use ANNs
|
42
|
+
}
|
43
|
+
|
44
|
+
if self.hnsw_ef != 0:
|
45
|
+
search_params["hnsw_ef"] = self.hnsw_ef
|
46
|
+
|
47
|
+
return search_params
|
@@ -0,0 +1,232 @@
|
|
1
|
+
"""Wrapper around the Qdrant over VectorDB"""
|
2
|
+
|
3
|
+
import logging
|
4
|
+
import time
|
5
|
+
from collections.abc import Iterable
|
6
|
+
from contextlib import contextmanager
|
7
|
+
|
8
|
+
from qdrant_client import QdrantClient
|
9
|
+
from qdrant_client.http.models import (
|
10
|
+
Batch,
|
11
|
+
CollectionStatus,
|
12
|
+
FieldCondition,
|
13
|
+
Filter,
|
14
|
+
HnswConfigDiff,
|
15
|
+
OptimizersConfigDiff,
|
16
|
+
PayloadSchemaType,
|
17
|
+
Range,
|
18
|
+
SearchParams,
|
19
|
+
VectorParams,
|
20
|
+
)
|
21
|
+
|
22
|
+
from ..api import VectorDB
|
23
|
+
from .config import QdrantLocalIndexConfig
|
24
|
+
|
25
|
+
log = logging.getLogger(__name__)
|
26
|
+
|
27
|
+
SECONDS_WAITING_FOR_INDEXING_API_CALL = 5
|
28
|
+
QDRANT_BATCH_SIZE = 100
|
29
|
+
|
30
|
+
|
31
|
+
def qdrant_collection_exists(client: QdrantClient, collection_name: str) -> bool:
|
32
|
+
collection_exists = True
|
33
|
+
|
34
|
+
try:
|
35
|
+
client.get_collection(collection_name)
|
36
|
+
except Exception:
|
37
|
+
collection_exists = False
|
38
|
+
|
39
|
+
return collection_exists
|
40
|
+
|
41
|
+
|
42
|
+
class QdrantLocal(VectorDB):
|
43
|
+
def __init__(
|
44
|
+
self,
|
45
|
+
dim: int,
|
46
|
+
db_config: dict,
|
47
|
+
db_case_config: QdrantLocalIndexConfig,
|
48
|
+
collection_name: str = "QdrantLocalCollection",
|
49
|
+
drop_old: bool = False,
|
50
|
+
name: str = "QdrantLocal",
|
51
|
+
**kwargs,
|
52
|
+
):
|
53
|
+
"""Initialize wrapper around the qdrant."""
|
54
|
+
self.name = name
|
55
|
+
self.db_config = db_config
|
56
|
+
self.case_config = db_case_config
|
57
|
+
self.search_parameter = self.case_config.search_param()
|
58
|
+
self.collection_name = collection_name
|
59
|
+
self.client = None
|
60
|
+
|
61
|
+
self._primary_field = "pk"
|
62
|
+
self._vector_field = "vector"
|
63
|
+
|
64
|
+
client = QdrantClient(**self.db_config)
|
65
|
+
|
66
|
+
# Lets just print the parameters here for double check
|
67
|
+
log.info(f"Case config: {self.case_config.index_param()}")
|
68
|
+
log.info(f"Search parameter: {self.search_parameter}")
|
69
|
+
|
70
|
+
if drop_old and qdrant_collection_exists(client, self.collection_name):
|
71
|
+
log.info(f"{self.name} client drop_old collection: {self.collection_name}")
|
72
|
+
client.delete_collection(self.collection_name)
|
73
|
+
|
74
|
+
if not qdrant_collection_exists(client, self.collection_name):
|
75
|
+
log.info(f"{self.name} create collection: {self.collection_name}")
|
76
|
+
self._create_collection(dim, client)
|
77
|
+
|
78
|
+
client = None
|
79
|
+
|
80
|
+
@contextmanager
|
81
|
+
def init(self):
|
82
|
+
"""
|
83
|
+
Examples:
|
84
|
+
>>> with self.init():
|
85
|
+
>>> self.insert_embeddings()
|
86
|
+
>>> self.search_embedding()
|
87
|
+
"""
|
88
|
+
# create connection
|
89
|
+
self.client = QdrantClient(**self.db_config)
|
90
|
+
yield
|
91
|
+
self.client = None
|
92
|
+
del self.client
|
93
|
+
|
94
|
+
def _create_collection(self, dim: int, qdrant_client: QdrantClient):
|
95
|
+
log.info(f"Create collection: {self.collection_name}")
|
96
|
+
log.info(
|
97
|
+
f"Index parameters: m={self.case_config.index_param()['m']}, "
|
98
|
+
f"ef_construct={self.case_config.index_param()['ef_construct']}, "
|
99
|
+
f"on_disk={self.case_config.index_param()['on_disk']}"
|
100
|
+
)
|
101
|
+
|
102
|
+
# If the on_disk is true, we enable both on disk index and vectors.
|
103
|
+
try:
|
104
|
+
qdrant_client.create_collection(
|
105
|
+
collection_name=self.collection_name,
|
106
|
+
vectors_config=VectorParams(
|
107
|
+
size=dim,
|
108
|
+
distance=self.case_config.index_param()["distance"],
|
109
|
+
on_disk=self.case_config.index_param()["on_disk"],
|
110
|
+
),
|
111
|
+
hnsw_config=HnswConfigDiff(
|
112
|
+
m=self.case_config.index_param()["m"],
|
113
|
+
ef_construct=self.case_config.index_param()["ef_construct"],
|
114
|
+
on_disk=self.case_config.index_param()["on_disk"],
|
115
|
+
),
|
116
|
+
)
|
117
|
+
|
118
|
+
qdrant_client.create_payload_index(
|
119
|
+
collection_name=self.collection_name,
|
120
|
+
field_name=self._primary_field,
|
121
|
+
field_schema=PayloadSchemaType.INTEGER,
|
122
|
+
)
|
123
|
+
|
124
|
+
except Exception as e:
|
125
|
+
if "already exists!" in str(e):
|
126
|
+
return
|
127
|
+
log.warning(f"Failed to create collection: {self.collection_name} error: {e}")
|
128
|
+
raise e from None
|
129
|
+
|
130
|
+
def optimize(self, data_size: int | None = None):
|
131
|
+
assert self.client, "Please call self.init() before"
|
132
|
+
# wait for vectors to be fully indexed
|
133
|
+
try:
|
134
|
+
while True:
|
135
|
+
info = self.client.get_collection(self.collection_name)
|
136
|
+
time.sleep(SECONDS_WAITING_FOR_INDEXING_API_CALL)
|
137
|
+
if info.status != CollectionStatus.GREEN:
|
138
|
+
continue
|
139
|
+
if info.status == CollectionStatus.GREEN:
|
140
|
+
log.info(f"Finishing building index for collection: {self.collection_name}")
|
141
|
+
msg = (
|
142
|
+
f"Stored vectors: {info.vectors_count}, Indexed vectors: {info.indexed_vectors_count}, "
|
143
|
+
f"Collection status: {info.indexed_vectors_count}"
|
144
|
+
)
|
145
|
+
log.info(msg)
|
146
|
+
return
|
147
|
+
|
148
|
+
except Exception as e:
|
149
|
+
log.warning(f"QdrantCloud ready to search error: {e}")
|
150
|
+
raise e from None
|
151
|
+
|
152
|
+
def insert_embeddings(
|
153
|
+
self,
|
154
|
+
embeddings: Iterable[list[float]],
|
155
|
+
metadata: list[int],
|
156
|
+
**kwargs,
|
157
|
+
) -> tuple[int, Exception]:
|
158
|
+
"""Insert embeddings into the database.
|
159
|
+
|
160
|
+
Args:
|
161
|
+
embeddings(list[list[float]]): list of embeddings
|
162
|
+
metadata(list[int]): list of metadata
|
163
|
+
kwargs: other arguments
|
164
|
+
|
165
|
+
Returns:
|
166
|
+
tuple[int, Exception]: number of embeddings inserted and exception if any
|
167
|
+
"""
|
168
|
+
assert self.client is not None
|
169
|
+
assert len(embeddings) == len(metadata)
|
170
|
+
insert_count = 0
|
171
|
+
|
172
|
+
# disable indexing for quick insertion
|
173
|
+
self.client.update_collection(
|
174
|
+
collection_name=self.collection_name,
|
175
|
+
optimizer_config=OptimizersConfigDiff(indexing_threshold=0),
|
176
|
+
)
|
177
|
+
try:
|
178
|
+
for offset in range(0, len(embeddings), QDRANT_BATCH_SIZE):
|
179
|
+
vectors = embeddings[offset : offset + QDRANT_BATCH_SIZE]
|
180
|
+
ids = metadata[offset : offset + QDRANT_BATCH_SIZE]
|
181
|
+
payloads = [{self._primary_field: v} for v in ids]
|
182
|
+
_ = self.client.upsert(
|
183
|
+
collection_name=self.collection_name,
|
184
|
+
wait=True,
|
185
|
+
points=Batch(ids=ids, payloads=payloads, vectors=vectors),
|
186
|
+
)
|
187
|
+
insert_count += QDRANT_BATCH_SIZE
|
188
|
+
# enable indexing after insertion
|
189
|
+
self.client.update_collection(
|
190
|
+
collection_name=self.collection_name,
|
191
|
+
optimizer_config=OptimizersConfigDiff(indexing_threshold=100),
|
192
|
+
)
|
193
|
+
|
194
|
+
except Exception as e:
|
195
|
+
log.info(f"Failed to insert data, {e}")
|
196
|
+
return insert_count, e
|
197
|
+
else:
|
198
|
+
return insert_count, None
|
199
|
+
|
200
|
+
def search_embedding(
|
201
|
+
self,
|
202
|
+
query: list[float],
|
203
|
+
k: int = 100,
|
204
|
+
filters: dict | None = None,
|
205
|
+
timeout: int | None = None,
|
206
|
+
) -> list[int]:
|
207
|
+
"""Perform a search on a query embedding and return results with score.
|
208
|
+
Should call self.init() first.
|
209
|
+
"""
|
210
|
+
assert self.client is not None
|
211
|
+
|
212
|
+
f = None
|
213
|
+
if filters:
|
214
|
+
f = Filter(
|
215
|
+
must=[
|
216
|
+
FieldCondition(
|
217
|
+
key=self._primary_field,
|
218
|
+
range=Range(
|
219
|
+
gt=filters.get("id"),
|
220
|
+
),
|
221
|
+
),
|
222
|
+
],
|
223
|
+
)
|
224
|
+
res = self.client.query_points(
|
225
|
+
collection_name=self.collection_name,
|
226
|
+
query=query,
|
227
|
+
limit=k,
|
228
|
+
query_filter=f,
|
229
|
+
search_params=SearchParams(**self.search_parameter),
|
230
|
+
).points
|
231
|
+
|
232
|
+
return [result.id for result in res]
|
@@ -15,12 +15,33 @@ from .. import DB
|
|
15
15
|
class WeaviateTypedDict(CommonTypedDict):
|
16
16
|
api_key: Annotated[
|
17
17
|
str,
|
18
|
-
click.option("--api-key", type=str, help="Weaviate api key", required=
|
18
|
+
click.option("--api-key", type=str, help="Weaviate api key", required=False, default=""),
|
19
19
|
]
|
20
20
|
url: Annotated[
|
21
21
|
str,
|
22
22
|
click.option("--url", type=str, help="Weaviate url", required=True),
|
23
23
|
]
|
24
|
+
no_auth: Annotated[
|
25
|
+
bool,
|
26
|
+
click.option(
|
27
|
+
"--no-auth",
|
28
|
+
is_flag=True,
|
29
|
+
help="Do not use api-key, set it to true if you are using a local setup. Default is False.",
|
30
|
+
default=False,
|
31
|
+
),
|
32
|
+
]
|
33
|
+
m: Annotated[
|
34
|
+
int,
|
35
|
+
click.option("--m", type=int, default=16, help="HNSW index parameter m."),
|
36
|
+
]
|
37
|
+
ef_construct: Annotated[
|
38
|
+
int,
|
39
|
+
click.option("--ef-construction", type=int, default=256, help="HNSW index parameter ef_construction"),
|
40
|
+
]
|
41
|
+
ef: Annotated[
|
42
|
+
int,
|
43
|
+
click.option("--ef", type=int, default=256, help="HNSW index parameter ef for search"),
|
44
|
+
]
|
24
45
|
|
25
46
|
|
26
47
|
@cli.command()
|
@@ -32,9 +53,14 @@ def Weaviate(**parameters: Unpack[WeaviateTypedDict]):
|
|
32
53
|
db=DB.WeaviateCloud,
|
33
54
|
db_config=WeaviateConfig(
|
34
55
|
db_label=parameters["db_label"],
|
35
|
-
api_key=SecretStr(parameters["api_key"]),
|
56
|
+
api_key=SecretStr(parameters["api_key"]) if parameters["api_key"] != "" else SecretStr("-"),
|
36
57
|
url=SecretStr(parameters["url"]),
|
58
|
+
no_auth=parameters["no_auth"],
|
59
|
+
),
|
60
|
+
db_case_config=WeaviateIndexConfig(
|
61
|
+
efConstruction=parameters["ef_construction"],
|
62
|
+
maxConnections=parameters["m"],
|
63
|
+
ef=parameters["ef"],
|
37
64
|
),
|
38
|
-
db_case_config=WeaviateIndexConfig(ef=256, efConstruction=256, maxConnections=16),
|
39
65
|
**parameters,
|
40
66
|
)
|
@@ -6,11 +6,13 @@ from ..api import DBCaseConfig, DBConfig, MetricType
|
|
6
6
|
class WeaviateConfig(DBConfig):
|
7
7
|
url: SecretStr
|
8
8
|
api_key: SecretStr
|
9
|
+
no_auth: bool | None = False
|
9
10
|
|
10
11
|
def to_dict(self) -> dict:
|
11
12
|
return {
|
12
13
|
"url": self.url.get_secret_value(),
|
13
14
|
"auth_client_secret": self.api_key.get_secret_value(),
|
15
|
+
"no_auth": self.no_auth,
|
14
16
|
}
|
15
17
|
|
16
18
|
|
@@ -38,6 +38,11 @@ class WeaviateCloud(VectorDB):
|
|
38
38
|
self._vector_field = "vector"
|
39
39
|
self._index_name = "vector_idx"
|
40
40
|
|
41
|
+
# If local setup is used, we
|
42
|
+
if db_config["no_auth"]:
|
43
|
+
del db_config["auth_client_secret"]
|
44
|
+
del db_config["no_auth"]
|
45
|
+
|
41
46
|
from weaviate import Client
|
42
47
|
|
43
48
|
client = Client(**db_config)
|