vectordb-bench 0.0.19__py3-none-any.whl → 0.0.21__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vectordb_bench/__init__.py +49 -24
- vectordb_bench/__main__.py +4 -3
- vectordb_bench/backend/assembler.py +12 -13
- vectordb_bench/backend/cases.py +55 -45
- vectordb_bench/backend/clients/__init__.py +85 -14
- vectordb_bench/backend/clients/aliyun_elasticsearch/aliyun_elasticsearch.py +1 -2
- vectordb_bench/backend/clients/aliyun_elasticsearch/config.py +3 -4
- vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py +112 -77
- vectordb_bench/backend/clients/aliyun_opensearch/config.py +6 -7
- vectordb_bench/backend/clients/alloydb/alloydb.py +59 -84
- vectordb_bench/backend/clients/alloydb/cli.py +51 -34
- vectordb_bench/backend/clients/alloydb/config.py +30 -30
- vectordb_bench/backend/clients/api.py +13 -24
- vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +50 -54
- vectordb_bench/backend/clients/aws_opensearch/cli.py +4 -7
- vectordb_bench/backend/clients/aws_opensearch/config.py +13 -9
- vectordb_bench/backend/clients/aws_opensearch/run.py +69 -59
- vectordb_bench/backend/clients/chroma/chroma.py +39 -40
- vectordb_bench/backend/clients/chroma/config.py +4 -2
- vectordb_bench/backend/clients/elastic_cloud/config.py +5 -5
- vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py +24 -26
- vectordb_bench/backend/clients/memorydb/cli.py +8 -8
- vectordb_bench/backend/clients/memorydb/config.py +2 -2
- vectordb_bench/backend/clients/memorydb/memorydb.py +67 -58
- vectordb_bench/backend/clients/milvus/cli.py +41 -83
- vectordb_bench/backend/clients/milvus/config.py +18 -8
- vectordb_bench/backend/clients/milvus/milvus.py +19 -39
- vectordb_bench/backend/clients/pgdiskann/cli.py +29 -22
- vectordb_bench/backend/clients/pgdiskann/config.py +29 -26
- vectordb_bench/backend/clients/pgdiskann/pgdiskann.py +56 -77
- vectordb_bench/backend/clients/pgvecto_rs/cli.py +9 -11
- vectordb_bench/backend/clients/pgvecto_rs/config.py +8 -14
- vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py +34 -43
- vectordb_bench/backend/clients/pgvector/cli.py +40 -31
- vectordb_bench/backend/clients/pgvector/config.py +63 -73
- vectordb_bench/backend/clients/pgvector/pgvector.py +98 -104
- vectordb_bench/backend/clients/pgvectorscale/cli.py +38 -24
- vectordb_bench/backend/clients/pgvectorscale/config.py +14 -15
- vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py +39 -49
- vectordb_bench/backend/clients/pinecone/config.py +1 -0
- vectordb_bench/backend/clients/pinecone/pinecone.py +15 -25
- vectordb_bench/backend/clients/qdrant_cloud/config.py +11 -10
- vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py +41 -35
- vectordb_bench/backend/clients/redis/cli.py +6 -12
- vectordb_bench/backend/clients/redis/config.py +7 -5
- vectordb_bench/backend/clients/redis/redis.py +95 -62
- vectordb_bench/backend/clients/test/cli.py +2 -3
- vectordb_bench/backend/clients/test/config.py +2 -2
- vectordb_bench/backend/clients/test/test.py +5 -9
- vectordb_bench/backend/clients/weaviate_cloud/cli.py +3 -4
- vectordb_bench/backend/clients/weaviate_cloud/config.py +2 -2
- vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +37 -26
- vectordb_bench/backend/clients/zilliz_cloud/cli.py +14 -11
- vectordb_bench/backend/clients/zilliz_cloud/config.py +2 -4
- vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py +1 -1
- vectordb_bench/backend/data_source.py +18 -14
- vectordb_bench/backend/dataset.py +47 -27
- vectordb_bench/backend/result_collector.py +2 -3
- vectordb_bench/backend/runner/__init__.py +4 -6
- vectordb_bench/backend/runner/mp_runner.py +56 -23
- vectordb_bench/backend/runner/rate_runner.py +30 -19
- vectordb_bench/backend/runner/read_write_runner.py +46 -22
- vectordb_bench/backend/runner/serial_runner.py +81 -46
- vectordb_bench/backend/runner/util.py +4 -3
- vectordb_bench/backend/task_runner.py +92 -92
- vectordb_bench/backend/utils.py +17 -10
- vectordb_bench/base.py +0 -1
- vectordb_bench/cli/cli.py +65 -60
- vectordb_bench/cli/vectordbbench.py +6 -7
- vectordb_bench/frontend/components/check_results/charts.py +8 -19
- vectordb_bench/frontend/components/check_results/data.py +4 -16
- vectordb_bench/frontend/components/check_results/filters.py +8 -16
- vectordb_bench/frontend/components/check_results/nav.py +4 -4
- vectordb_bench/frontend/components/check_results/priceTable.py +1 -3
- vectordb_bench/frontend/components/check_results/stPageConfig.py +2 -1
- vectordb_bench/frontend/components/concurrent/charts.py +12 -12
- vectordb_bench/frontend/components/custom/displayCustomCase.py +17 -11
- vectordb_bench/frontend/components/custom/displaypPrams.py +4 -2
- vectordb_bench/frontend/components/custom/getCustomConfig.py +1 -2
- vectordb_bench/frontend/components/custom/initStyle.py +1 -1
- vectordb_bench/frontend/components/get_results/saveAsImage.py +2 -0
- vectordb_bench/frontend/components/run_test/caseSelector.py +3 -9
- vectordb_bench/frontend/components/run_test/dbConfigSetting.py +1 -4
- vectordb_bench/frontend/components/run_test/dbSelector.py +1 -1
- vectordb_bench/frontend/components/run_test/generateTasks.py +8 -8
- vectordb_bench/frontend/components/run_test/submitTask.py +14 -18
- vectordb_bench/frontend/components/tables/data.py +3 -6
- vectordb_bench/frontend/config/dbCaseConfigs.py +51 -84
- vectordb_bench/frontend/pages/concurrent.py +3 -5
- vectordb_bench/frontend/pages/custom.py +30 -9
- vectordb_bench/frontend/pages/quries_per_dollar.py +3 -3
- vectordb_bench/frontend/pages/run_test.py +3 -7
- vectordb_bench/frontend/utils.py +1 -1
- vectordb_bench/frontend/vdb_benchmark.py +4 -6
- vectordb_bench/interface.py +45 -24
- vectordb_bench/log_util.py +59 -64
- vectordb_bench/metric.py +10 -11
- vectordb_bench/models.py +26 -43
- {vectordb_bench-0.0.19.dist-info → vectordb_bench-0.0.21.dist-info}/METADATA +22 -15
- vectordb_bench-0.0.21.dist-info/RECORD +135 -0
- {vectordb_bench-0.0.19.dist-info → vectordb_bench-0.0.21.dist-info}/WHEEL +1 -1
- vectordb_bench-0.0.19.dist-info/RECORD +0 -135
- {vectordb_bench-0.0.19.dist-info → vectordb_bench-0.0.21.dist-info}/LICENSE +0 -0
- {vectordb_bench-0.0.19.dist-info → vectordb_bench-0.0.21.dist-info}/entry_points.txt +0 -0
- {vectordb_bench-0.0.19.dist-info → vectordb_bench-0.0.21.dist-info}/top_level.txt +0 -0
@@ -32,8 +32,7 @@ def get_custom_configs():
|
|
32
32
|
|
33
33
|
def save_custom_configs(custom_configs: list[CustomDatasetConfig]):
|
34
34
|
with open(config.CUSTOM_CONFIG_DIR, "w") as f:
|
35
|
-
json.dump([custom_config.dict()
|
36
|
-
for custom_config in custom_configs], f, indent=4)
|
35
|
+
json.dump([custom_config.dict() for custom_config in custom_configs], f, indent=4)
|
37
36
|
|
38
37
|
|
39
38
|
def generate_custom_case():
|
@@ -9,10 +9,12 @@ HTML_2_CANVAS_URL = "https://unpkg.com/html2canvas@1.4.1/dist/html2canvas.js"
|
|
9
9
|
def load_unpkg(src: str) -> str:
|
10
10
|
return requests.get(src).text
|
11
11
|
|
12
|
+
|
12
13
|
def getResults(container, pageName="vectordb_bench"):
|
13
14
|
container.subheader("Get results")
|
14
15
|
saveAsImage(container, pageName)
|
15
16
|
|
17
|
+
|
16
18
|
def saveAsImage(container, pageName):
|
17
19
|
html2canvasJS = load_unpkg(HTML_2_CANVAS_URL)
|
18
20
|
container.write()
|
@@ -1,6 +1,4 @@
|
|
1
|
-
|
2
1
|
from vectordb_bench.frontend.config.styles import *
|
3
|
-
from vectordb_bench.backend.cases import CaseType
|
4
2
|
from vectordb_bench.frontend.config.dbCaseConfigs import *
|
5
3
|
from collections import defaultdict
|
6
4
|
|
@@ -23,8 +21,7 @@ def caseSelector(st, activedDbList: list[DB]):
|
|
23
21
|
dbToCaseConfigs = defaultdict(lambda: defaultdict(dict))
|
24
22
|
caseClusters = UI_CASE_CLUSTERS + [get_custom_case_cluter()]
|
25
23
|
for caseCluster in caseClusters:
|
26
|
-
activedCaseList += caseClusterExpander(
|
27
|
-
st, caseCluster, dbToCaseClusterConfigs, activedDbList)
|
24
|
+
activedCaseList += caseClusterExpander(st, caseCluster, dbToCaseClusterConfigs, activedDbList)
|
28
25
|
for db in dbToCaseClusterConfigs:
|
29
26
|
for uiCaseItem in dbToCaseClusterConfigs[db]:
|
30
27
|
for case in uiCaseItem.cases:
|
@@ -40,8 +37,7 @@ def caseClusterExpander(st, caseCluster: UICaseItemCluster, dbToCaseClusterConfi
|
|
40
37
|
if uiCaseItem.isLine:
|
41
38
|
addHorizontalLine(expander)
|
42
39
|
else:
|
43
|
-
activedCases += caseItemCheckbox(expander,
|
44
|
-
dbToCaseClusterConfigs, uiCaseItem, activedDbList)
|
40
|
+
activedCases += caseItemCheckbox(expander, dbToCaseClusterConfigs, uiCaseItem, activedDbList)
|
45
41
|
return activedCases
|
46
42
|
|
47
43
|
|
@@ -53,9 +49,7 @@ def caseItemCheckbox(st, dbToCaseClusterConfigs, uiCaseItem: UICaseItem, actived
|
|
53
49
|
)
|
54
50
|
|
55
51
|
if selected:
|
56
|
-
caseConfigSetting(
|
57
|
-
st.container(), dbToCaseClusterConfigs, uiCaseItem, activedDbList
|
58
|
-
)
|
52
|
+
caseConfigSetting(st.container(), dbToCaseClusterConfigs, uiCaseItem, activedDbList)
|
59
53
|
|
60
54
|
return uiCaseItem.cases if selected else []
|
61
55
|
|
@@ -42,10 +42,7 @@ def dbConfigSettingItem(st, activeDb: DB):
|
|
42
42
|
|
43
43
|
# db config (unique)
|
44
44
|
for key, property in properties.items():
|
45
|
-
if (
|
46
|
-
key not in dbConfigClass.common_short_configs()
|
47
|
-
and key not in dbConfigClass.common_long_configs()
|
48
|
-
):
|
45
|
+
if key not in dbConfigClass.common_short_configs() and key not in dbConfigClass.common_long_configs():
|
49
46
|
column = columns[idx % DB_CONFIG_SETTING_COLUMNS]
|
50
47
|
idx += 1
|
51
48
|
dbConfig[key] = column.text_input(
|
@@ -22,7 +22,7 @@ def dbSelector(st):
|
|
22
22
|
dbIsActived[db] = column.checkbox(db.name)
|
23
23
|
try:
|
24
24
|
column.image(DB_TO_ICON.get(db, ""))
|
25
|
-
except MediaFileStorageError
|
25
|
+
except MediaFileStorageError:
|
26
26
|
column.warning(f"{db.name} image not available")
|
27
27
|
pass
|
28
28
|
activedDbList = [db for db in DB_LIST if dbIsActived[db]]
|
@@ -7,13 +7,13 @@ def generate_tasks(activedDbList: list[DB], dbConfigs, activedCaseList: list[Cas
|
|
7
7
|
for db in activedDbList:
|
8
8
|
for case in activedCaseList:
|
9
9
|
task = TaskConfig(
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
10
|
+
db=db.value,
|
11
|
+
db_config=dbConfigs[db],
|
12
|
+
case_config=case,
|
13
|
+
db_case_config=db.case_config_cls(allCaseConfigs[db][case].get(CaseConfigParamType.IndexType, None))(
|
14
|
+
**{key.value: value for key, value in allCaseConfigs[db][case].items()}
|
15
|
+
),
|
16
|
+
)
|
17
17
|
tasks.append(task)
|
18
|
-
|
18
|
+
|
19
19
|
return tasks
|
@@ -1,6 +1,6 @@
|
|
1
1
|
from datetime import datetime
|
2
|
-
from vectordb_bench.frontend.config
|
3
|
-
from vectordb_bench.interface import
|
2
|
+
from vectordb_bench.frontend.config import styles
|
3
|
+
from vectordb_bench.interface import benchmark_runner
|
4
4
|
|
5
5
|
|
6
6
|
def submitTask(st, tasks, isAllValid):
|
@@ -27,10 +27,8 @@ def submitTask(st, tasks, isAllValid):
|
|
27
27
|
|
28
28
|
def taskLabelInput(st):
|
29
29
|
defaultTaskLabel = datetime.now().strftime("%Y%m%d%H")
|
30
|
-
columns = st.columns(TASK_LABEL_INPUT_COLUMNS)
|
31
|
-
taskLabel = columns[0].text_input(
|
32
|
-
"task_label", defaultTaskLabel, label_visibility="collapsed"
|
33
|
-
)
|
30
|
+
columns = st.columns(styles.TASK_LABEL_INPUT_COLUMNS)
|
31
|
+
taskLabel = columns[0].text_input("task_label", defaultTaskLabel, label_visibility="collapsed")
|
34
32
|
return taskLabel
|
35
33
|
|
36
34
|
|
@@ -46,10 +44,8 @@ def advancedSettings(st):
|
|
46
44
|
)
|
47
45
|
|
48
46
|
container = st.columns([1, 2])
|
49
|
-
k = container[0].number_input("k",min_value=1, value=100, label_visibility="collapsed")
|
50
|
-
container[1].caption(
|
51
|
-
"K value for number of nearest neighbors to search"
|
52
|
-
)
|
47
|
+
k = container[0].number_input("k", min_value=1, value=100, label_visibility="collapsed")
|
48
|
+
container[1].caption("K value for number of nearest neighbors to search")
|
53
49
|
|
54
50
|
return index_already_exists, use_aliyun, k
|
55
51
|
|
@@ -58,20 +54,20 @@ def controlPanel(st, tasks, taskLabel, isAllValid):
|
|
58
54
|
index_already_exists, use_aliyun, k = advancedSettings(st)
|
59
55
|
|
60
56
|
def runHandler():
|
61
|
-
|
57
|
+
benchmark_runner.set_drop_old(not index_already_exists)
|
62
58
|
for task in tasks:
|
63
59
|
task.case_config.k = k
|
64
|
-
|
65
|
-
|
60
|
+
benchmark_runner.set_download_address(use_aliyun)
|
61
|
+
benchmark_runner.run(tasks, taskLabel)
|
66
62
|
|
67
63
|
def stopHandler():
|
68
|
-
|
64
|
+
benchmark_runner.stop_running()
|
69
65
|
|
70
|
-
isRunning =
|
66
|
+
isRunning = benchmark_runner.has_running()
|
71
67
|
|
72
68
|
if isRunning:
|
73
|
-
currentTaskId =
|
74
|
-
tasksCount =
|
69
|
+
currentTaskId = benchmark_runner.get_current_task_id()
|
70
|
+
tasksCount = benchmark_runner.get_tasks_count()
|
75
71
|
text = f":running: Running Task {currentTaskId} / {tasksCount}"
|
76
72
|
st.progress(currentTaskId / tasksCount, text=text)
|
77
73
|
|
@@ -89,7 +85,7 @@ def controlPanel(st, tasks, taskLabel, isAllValid):
|
|
89
85
|
)
|
90
86
|
|
91
87
|
else:
|
92
|
-
errorText =
|
88
|
+
errorText = benchmark_runner.latest_error or ""
|
93
89
|
if len(errorText) > 0:
|
94
90
|
st.error(errorText)
|
95
91
|
disabled = True if len(tasks) == 0 or not isAllValid else False
|
@@ -1,12 +1,11 @@
|
|
1
1
|
from dataclasses import asdict
|
2
|
-
from vectordb_bench.
|
3
|
-
from vectordb_bench.interface import benchMarkRunner
|
2
|
+
from vectordb_bench.interface import benchmark_runner
|
4
3
|
from vectordb_bench.models import CaseResult, ResultLabel
|
5
4
|
import pandas as pd
|
6
5
|
|
7
6
|
|
8
7
|
def getNewResults():
|
9
|
-
allResults =
|
8
|
+
allResults = benchmark_runner.get_results()
|
10
9
|
newResults: list[CaseResult] = []
|
11
10
|
|
12
11
|
for res in allResults:
|
@@ -14,7 +13,6 @@ def getNewResults():
|
|
14
13
|
for result in results:
|
15
14
|
if result.label == ResultLabel.NORMAL:
|
16
15
|
newResults.append(result)
|
17
|
-
|
18
16
|
|
19
17
|
df = pd.DataFrame(formatData(newResults))
|
20
18
|
return df
|
@@ -26,7 +24,6 @@ def formatData(caseResults: list[CaseResult]):
|
|
26
24
|
db = caseResult.task_config.db.value
|
27
25
|
db_label = caseResult.task_config.db_config.db_label
|
28
26
|
case_config = caseResult.task_config.case_config
|
29
|
-
db_case_config = caseResult.task_config.db_case_config
|
30
27
|
case = case_config.case_id.case_cls()
|
31
28
|
filter_rate = case.filter_rate
|
32
29
|
dataset = case.dataset.data.name
|
@@ -41,4 +38,4 @@ def formatData(caseResults: list[CaseResult]):
|
|
41
38
|
**metrics,
|
42
39
|
}
|
43
40
|
)
|
44
|
-
return data
|
41
|
+
return data
|
@@ -33,9 +33,9 @@ class UICaseItem(BaseModel):
|
|
33
33
|
def __init__(
|
34
34
|
self,
|
35
35
|
isLine: bool = False,
|
36
|
-
case_id: CaseType = None,
|
37
|
-
custom_case: dict =
|
38
|
-
cases: list[CaseConfig] =
|
36
|
+
case_id: CaseType | None = None,
|
37
|
+
custom_case: dict | None = None,
|
38
|
+
cases: list[CaseConfig] | None = None,
|
39
39
|
label: str = "",
|
40
40
|
description: str = "",
|
41
41
|
caseLabel: CaseLabel = CaseLabel.Performance,
|
@@ -70,17 +70,13 @@ class UICaseItemCluster(BaseModel):
|
|
70
70
|
def get_custom_case_items() -> list[UICaseItem]:
|
71
71
|
custom_configs = get_custom_configs()
|
72
72
|
return [
|
73
|
-
UICaseItem(
|
74
|
-
case_id=CaseType.PerformanceCustomDataset, custom_case=custom_config.dict()
|
75
|
-
)
|
73
|
+
UICaseItem(case_id=CaseType.PerformanceCustomDataset, custom_case=custom_config.dict())
|
76
74
|
for custom_config in custom_configs
|
77
75
|
]
|
78
76
|
|
79
77
|
|
80
78
|
def get_custom_case_cluter() -> UICaseItemCluster:
|
81
|
-
return UICaseItemCluster(
|
82
|
-
label="Custom Search Performance Test", uiCaseItems=get_custom_case_items()
|
83
|
-
)
|
79
|
+
return UICaseItemCluster(label="Custom Search Performance Test", uiCaseItems=get_custom_case_items())
|
84
80
|
|
85
81
|
|
86
82
|
UI_CASE_CLUSTERS: list[UICaseItemCluster] = [
|
@@ -224,8 +220,7 @@ CaseConfigParamInput_max_neighbors = CaseConfigInput(
|
|
224
220
|
"max": 300,
|
225
221
|
"value": 32,
|
226
222
|
},
|
227
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
228
|
-
== IndexType.DISKANN.value,
|
223
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.DISKANN.value,
|
229
224
|
)
|
230
225
|
|
231
226
|
CaseConfigParamInput_l_value_ib = CaseConfigInput(
|
@@ -236,8 +231,7 @@ CaseConfigParamInput_l_value_ib = CaseConfigInput(
|
|
236
231
|
"max": 300,
|
237
232
|
"value": 50,
|
238
233
|
},
|
239
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
240
|
-
== IndexType.DISKANN.value,
|
234
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.DISKANN.value,
|
241
235
|
)
|
242
236
|
|
243
237
|
CaseConfigParamInput_l_value_is = CaseConfigInput(
|
@@ -248,8 +242,7 @@ CaseConfigParamInput_l_value_is = CaseConfigInput(
|
|
248
242
|
"max": 300,
|
249
243
|
"value": 40,
|
250
244
|
},
|
251
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
252
|
-
== IndexType.DISKANN.value,
|
245
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.DISKANN.value,
|
253
246
|
)
|
254
247
|
|
255
248
|
CaseConfigParamInput_num_neighbors = CaseConfigInput(
|
@@ -260,8 +253,7 @@ CaseConfigParamInput_num_neighbors = CaseConfigInput(
|
|
260
253
|
"max": 300,
|
261
254
|
"value": 50,
|
262
255
|
},
|
263
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
264
|
-
== IndexType.STREAMING_DISKANN.value,
|
256
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.STREAMING_DISKANN.value,
|
265
257
|
)
|
266
258
|
|
267
259
|
CaseConfigParamInput_search_list_size = CaseConfigInput(
|
@@ -272,8 +264,7 @@ CaseConfigParamInput_search_list_size = CaseConfigInput(
|
|
272
264
|
"max": 300,
|
273
265
|
"value": 100,
|
274
266
|
},
|
275
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
276
|
-
== IndexType.STREAMING_DISKANN.value,
|
267
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.STREAMING_DISKANN.value,
|
277
268
|
)
|
278
269
|
|
279
270
|
CaseConfigParamInput_max_alpha = CaseConfigInput(
|
@@ -284,8 +275,7 @@ CaseConfigParamInput_max_alpha = CaseConfigInput(
|
|
284
275
|
"max": 2.0,
|
285
276
|
"value": 1.2,
|
286
277
|
},
|
287
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
288
|
-
== IndexType.STREAMING_DISKANN.value,
|
278
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.STREAMING_DISKANN.value,
|
289
279
|
)
|
290
280
|
|
291
281
|
CaseConfigParamInput_num_dimensions = CaseConfigInput(
|
@@ -296,8 +286,7 @@ CaseConfigParamInput_num_dimensions = CaseConfigInput(
|
|
296
286
|
"max": 2000,
|
297
287
|
"value": 0,
|
298
288
|
},
|
299
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
300
|
-
== IndexType.STREAMING_DISKANN.value,
|
289
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.STREAMING_DISKANN.value,
|
301
290
|
)
|
302
291
|
|
303
292
|
CaseConfigParamInput_query_search_list_size = CaseConfigInput(
|
@@ -308,8 +297,7 @@ CaseConfigParamInput_query_search_list_size = CaseConfigInput(
|
|
308
297
|
"max": 150,
|
309
298
|
"value": 100,
|
310
299
|
},
|
311
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
312
|
-
== IndexType.STREAMING_DISKANN.value,
|
300
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.STREAMING_DISKANN.value,
|
313
301
|
)
|
314
302
|
|
315
303
|
|
@@ -321,8 +309,7 @@ CaseConfigParamInput_query_rescore = CaseConfigInput(
|
|
321
309
|
"max": 150,
|
322
310
|
"value": 50,
|
323
311
|
},
|
324
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
325
|
-
== IndexType.STREAMING_DISKANN.value,
|
312
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.STREAMING_DISKANN.value,
|
326
313
|
)
|
327
314
|
|
328
315
|
CaseConfigParamInput_IndexType_PgVector = CaseConfigInput(
|
@@ -358,8 +345,7 @@ CaseConfigParamInput_M = CaseConfigInput(
|
|
358
345
|
"max": 64,
|
359
346
|
"value": 30,
|
360
347
|
},
|
361
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
362
|
-
== IndexType.HNSW.value,
|
348
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value,
|
363
349
|
)
|
364
350
|
|
365
351
|
CaseConfigParamInput_m = CaseConfigInput(
|
@@ -370,8 +356,7 @@ CaseConfigParamInput_m = CaseConfigInput(
|
|
370
356
|
"max": 64,
|
371
357
|
"value": 16,
|
372
358
|
},
|
373
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
374
|
-
== IndexType.HNSW.value,
|
359
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value,
|
375
360
|
)
|
376
361
|
|
377
362
|
|
@@ -383,8 +368,7 @@ CaseConfigParamInput_EFConstruction_Milvus = CaseConfigInput(
|
|
383
368
|
"max": 512,
|
384
369
|
"value": 360,
|
385
370
|
},
|
386
|
-
isDisplayed=lambda config: config[CaseConfigParamType.IndexType]
|
387
|
-
== IndexType.HNSW.value,
|
371
|
+
isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value,
|
388
372
|
)
|
389
373
|
|
390
374
|
CaseConfigParamInput_EFConstruction_Weaviate = CaseConfigInput(
|
@@ -480,8 +464,7 @@ CaseConfigParamInput_EFConstruction_PgVectoRS = CaseConfigInput(
|
|
480
464
|
"max": 2000,
|
481
465
|
"value": 300,
|
482
466
|
},
|
483
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
484
|
-
== IndexType.HNSW.value,
|
467
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value,
|
485
468
|
)
|
486
469
|
|
487
470
|
CaseConfigParamInput_EFSearch_PgVectoRS = CaseConfigInput(
|
@@ -492,8 +475,7 @@ CaseConfigParamInput_EFSearch_PgVectoRS = CaseConfigInput(
|
|
492
475
|
"max": 65535,
|
493
476
|
"value": 100,
|
494
477
|
},
|
495
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
496
|
-
== IndexType.HNSW.value,
|
478
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value,
|
497
479
|
)
|
498
480
|
|
499
481
|
CaseConfigParamInput_EFConstruction_PgVector = CaseConfigInput(
|
@@ -504,8 +486,7 @@ CaseConfigParamInput_EFConstruction_PgVector = CaseConfigInput(
|
|
504
486
|
"max": 1024,
|
505
487
|
"value": 256,
|
506
488
|
},
|
507
|
-
isDisplayed=lambda config: config[CaseConfigParamType.IndexType]
|
508
|
-
== IndexType.HNSW.value,
|
489
|
+
isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value,
|
509
490
|
)
|
510
491
|
|
511
492
|
|
@@ -537,8 +518,7 @@ CaseConfigParamInput_EF_Milvus = CaseConfigInput(
|
|
537
518
|
"max": MAX_STREAMLIT_INT,
|
538
519
|
"value": 100,
|
539
520
|
},
|
540
|
-
isDisplayed=lambda config: config[CaseConfigParamType.IndexType]
|
541
|
-
== IndexType.HNSW.value,
|
521
|
+
isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value,
|
542
522
|
)
|
543
523
|
|
544
524
|
CaseConfigParamInput_EF_Weaviate = CaseConfigInput(
|
@@ -565,8 +545,7 @@ CaseConfigParamInput_SearchList = CaseConfigInput(
|
|
565
545
|
"max": MAX_STREAMLIT_INT,
|
566
546
|
"value": 100,
|
567
547
|
},
|
568
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
569
|
-
== IndexType.DISKANN.value,
|
548
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.DISKANN.value,
|
570
549
|
)
|
571
550
|
|
572
551
|
CaseConfigParamInput_Nlist = CaseConfigInput(
|
@@ -611,8 +590,7 @@ CaseConfigParamInput_M_PQ = CaseConfigInput(
|
|
611
590
|
"max": 65536,
|
612
591
|
"value": 0,
|
613
592
|
},
|
614
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
615
|
-
in [IndexType.GPU_IVF_PQ.value],
|
593
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_IVF_PQ.value],
|
616
594
|
)
|
617
595
|
|
618
596
|
|
@@ -624,8 +602,7 @@ CaseConfigParamInput_Nbits_PQ = CaseConfigInput(
|
|
624
602
|
"max": 65536,
|
625
603
|
"value": 8,
|
626
604
|
},
|
627
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
628
|
-
in [IndexType.GPU_IVF_PQ.value],
|
605
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_IVF_PQ.value],
|
629
606
|
)
|
630
607
|
|
631
608
|
CaseConfigParamInput_intermediate_graph_degree = CaseConfigInput(
|
@@ -636,8 +613,7 @@ CaseConfigParamInput_intermediate_graph_degree = CaseConfigInput(
|
|
636
613
|
"max": 65536,
|
637
614
|
"value": 64,
|
638
615
|
},
|
639
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
640
|
-
in [IndexType.GPU_CAGRA.value],
|
616
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value],
|
641
617
|
)
|
642
618
|
|
643
619
|
CaseConfigParamInput_graph_degree = CaseConfigInput(
|
@@ -648,8 +624,7 @@ CaseConfigParamInput_graph_degree = CaseConfigInput(
|
|
648
624
|
"max": 65536,
|
649
625
|
"value": 32,
|
650
626
|
},
|
651
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
652
|
-
in [IndexType.GPU_CAGRA.value],
|
627
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value],
|
653
628
|
)
|
654
629
|
|
655
630
|
CaseConfigParamInput_itopk_size = CaseConfigInput(
|
@@ -660,8 +635,7 @@ CaseConfigParamInput_itopk_size = CaseConfigInput(
|
|
660
635
|
"max": 65536,
|
661
636
|
"value": 128,
|
662
637
|
},
|
663
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
664
|
-
in [IndexType.GPU_CAGRA.value],
|
638
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value],
|
665
639
|
)
|
666
640
|
|
667
641
|
CaseConfigParamInput_team_size = CaseConfigInput(
|
@@ -672,8 +646,7 @@ CaseConfigParamInput_team_size = CaseConfigInput(
|
|
672
646
|
"max": 65536,
|
673
647
|
"value": 0,
|
674
648
|
},
|
675
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
676
|
-
in [IndexType.GPU_CAGRA.value],
|
649
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value],
|
677
650
|
)
|
678
651
|
|
679
652
|
CaseConfigParamInput_search_width = CaseConfigInput(
|
@@ -684,8 +657,7 @@ CaseConfigParamInput_search_width = CaseConfigInput(
|
|
684
657
|
"max": 65536,
|
685
658
|
"value": 4,
|
686
659
|
},
|
687
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
688
|
-
in [IndexType.GPU_CAGRA.value],
|
660
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value],
|
689
661
|
)
|
690
662
|
|
691
663
|
CaseConfigParamInput_min_iterations = CaseConfigInput(
|
@@ -696,8 +668,7 @@ CaseConfigParamInput_min_iterations = CaseConfigInput(
|
|
696
668
|
"max": 65536,
|
697
669
|
"value": 0,
|
698
670
|
},
|
699
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
700
|
-
in [IndexType.GPU_CAGRA.value],
|
671
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value],
|
701
672
|
)
|
702
673
|
|
703
674
|
CaseConfigParamInput_max_iterations = CaseConfigInput(
|
@@ -708,8 +679,7 @@ CaseConfigParamInput_max_iterations = CaseConfigInput(
|
|
708
679
|
"max": 65536,
|
709
680
|
"value": 0,
|
710
681
|
},
|
711
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
712
|
-
in [IndexType.GPU_CAGRA.value],
|
682
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value],
|
713
683
|
)
|
714
684
|
|
715
685
|
CaseConfigParamInput_build_algo = CaseConfigInput(
|
@@ -718,8 +688,7 @@ CaseConfigParamInput_build_algo = CaseConfigInput(
|
|
718
688
|
inputConfig={
|
719
689
|
"options": ["IVF_PQ", "NN_DESCENT"],
|
720
690
|
},
|
721
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
722
|
-
in [IndexType.GPU_CAGRA.value],
|
691
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value],
|
723
692
|
)
|
724
693
|
|
725
694
|
|
@@ -762,8 +731,7 @@ CaseConfigParamInput_Lists = CaseConfigInput(
|
|
762
731
|
"max": 65536,
|
763
732
|
"value": 10,
|
764
733
|
},
|
765
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
766
|
-
in [IndexType.IVFFlat.value],
|
734
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.IVFFlat.value],
|
767
735
|
)
|
768
736
|
|
769
737
|
CaseConfigParamInput_Probes = CaseConfigInput(
|
@@ -784,8 +752,7 @@ CaseConfigParamInput_Lists_PgVector = CaseConfigInput(
|
|
784
752
|
"max": 65536,
|
785
753
|
"value": 10,
|
786
754
|
},
|
787
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
788
|
-
== IndexType.IVFFlat.value,
|
755
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.IVFFlat.value,
|
789
756
|
)
|
790
757
|
|
791
758
|
CaseConfigParamInput_Probes_PgVector = CaseConfigInput(
|
@@ -796,8 +763,7 @@ CaseConfigParamInput_Probes_PgVector = CaseConfigInput(
|
|
796
763
|
"max": 65536,
|
797
764
|
"value": 1,
|
798
765
|
},
|
799
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
800
|
-
== IndexType.IVFFlat.value,
|
766
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.IVFFlat.value,
|
801
767
|
)
|
802
768
|
|
803
769
|
CaseConfigParamInput_EFSearch_PgVector = CaseConfigInput(
|
@@ -808,8 +774,7 @@ CaseConfigParamInput_EFSearch_PgVector = CaseConfigInput(
|
|
808
774
|
"max": 2048,
|
809
775
|
"value": 256,
|
810
776
|
},
|
811
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
812
|
-
== IndexType.HNSW.value,
|
777
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value,
|
813
778
|
)
|
814
779
|
|
815
780
|
|
@@ -845,8 +810,7 @@ CaseConfigParamInput_QuantizationRatio_PgVectoRS = CaseConfigInput(
|
|
845
810
|
inputConfig={
|
846
811
|
"options": ["x4", "x8", "x16", "x32", "x64"],
|
847
812
|
},
|
848
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None)
|
849
|
-
== "product"
|
813
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) == "product"
|
850
814
|
and config.get(CaseConfigParamType.IndexType, None)
|
851
815
|
in [
|
852
816
|
IndexType.HNSW.value,
|
@@ -885,8 +849,7 @@ CaseConfigParamInput_reranking_PgVector = CaseConfigInput(
|
|
885
849
|
inputConfig={
|
886
850
|
"value": False,
|
887
851
|
},
|
888
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None)
|
889
|
-
== "bit"
|
852
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) == "bit",
|
890
853
|
)
|
891
854
|
|
892
855
|
CaseConfigParamInput_quantized_fetch_limit_PgVector = CaseConfigInput(
|
@@ -899,8 +862,8 @@ CaseConfigParamInput_quantized_fetch_limit_PgVector = CaseConfigInput(
|
|
899
862
|
"max": 1000,
|
900
863
|
"value": 200,
|
901
864
|
},
|
902
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None)
|
903
|
-
|
865
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) == "bit"
|
866
|
+
and config.get(CaseConfigParamType.reranking, False),
|
904
867
|
)
|
905
868
|
|
906
869
|
|
@@ -908,12 +871,10 @@ CaseConfigParamInput_reranking_metric_PgVector = CaseConfigInput(
|
|
908
871
|
label=CaseConfigParamType.rerankingMetric,
|
909
872
|
inputType=InputType.Option,
|
910
873
|
inputConfig={
|
911
|
-
"options": [
|
912
|
-
metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD"]
|
913
|
-
],
|
874
|
+
"options": [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD"]],
|
914
875
|
},
|
915
|
-
isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None)
|
916
|
-
|
876
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) == "bit"
|
877
|
+
and config.get(CaseConfigParamType.reranking, False),
|
917
878
|
)
|
918
879
|
|
919
880
|
|
@@ -1131,7 +1092,10 @@ ESPerformanceConfig = [
|
|
1131
1092
|
CaseConfigParamInput_NumCandidates_ES,
|
1132
1093
|
]
|
1133
1094
|
|
1134
|
-
AWSOpensearchLoadingConfig = [
|
1095
|
+
AWSOpensearchLoadingConfig = [
|
1096
|
+
CaseConfigParamInput_EFConstruction_AWSOpensearch,
|
1097
|
+
CaseConfigParamInput_M_AWSOpensearch,
|
1098
|
+
]
|
1135
1099
|
AWSOpenSearchPerformanceConfig = [
|
1136
1100
|
CaseConfigParamInput_EFConstruction_AWSOpensearch,
|
1137
1101
|
CaseConfigParamInput_M_AWSOpensearch,
|
@@ -1250,7 +1214,10 @@ AlloyDBPerformanceConfig = [
|
|
1250
1214
|
CaseConfigParamInput_max_parallel_workers_AlloyDB,
|
1251
1215
|
]
|
1252
1216
|
|
1253
|
-
AliyunElasticsearchLoadingConfig = [
|
1217
|
+
AliyunElasticsearchLoadingConfig = [
|
1218
|
+
CaseConfigParamInput_EFConstruction_AliES,
|
1219
|
+
CaseConfigParamInput_M_AliES,
|
1220
|
+
]
|
1254
1221
|
AliyunElasticsearchPerformanceConfig = [
|
1255
1222
|
CaseConfigParamInput_EFConstruction_AliES,
|
1256
1223
|
CaseConfigParamInput_M_AliES,
|
@@ -9,7 +9,7 @@ from vectordb_bench.frontend.components.check_results.filters import getshownDat
|
|
9
9
|
from vectordb_bench.frontend.components.concurrent.charts import drawChartsByCase
|
10
10
|
from vectordb_bench.frontend.components.get_results.saveAsImage import getResults
|
11
11
|
from vectordb_bench.frontend.config.styles import FAVICON
|
12
|
-
from vectordb_bench.interface import
|
12
|
+
from vectordb_bench.interface import benchmark_runner
|
13
13
|
from vectordb_bench.models import TestResult
|
14
14
|
|
15
15
|
|
@@ -25,7 +25,7 @@ def main():
|
|
25
25
|
# header
|
26
26
|
drawHeaderIcon(st)
|
27
27
|
|
28
|
-
allResults =
|
28
|
+
allResults = benchmark_runner.get_results()
|
29
29
|
|
30
30
|
def check_conc_data(res: TestResult):
|
31
31
|
case_results = res.results
|
@@ -57,9 +57,7 @@ def main():
|
|
57
57
|
|
58
58
|
# main
|
59
59
|
latency_type = st.radio("Latency Type", options=["latency_p99", "latency_avg"])
|
60
|
-
drawChartsByCase(
|
61
|
-
shownData, showCaseNames, st.container(), latency_type=latency_type
|
62
|
-
)
|
60
|
+
drawChartsByCase(shownData, showCaseNames, st.container(), latency_type=latency_type)
|
63
61
|
|
64
62
|
# footer
|
65
63
|
footer(st.container())
|