vectordb-bench 0.0.18__py3-none-any.whl → 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vectordb_bench/__init__.py +49 -24
- vectordb_bench/__main__.py +4 -3
- vectordb_bench/backend/assembler.py +12 -13
- vectordb_bench/backend/cases.py +56 -46
- vectordb_bench/backend/clients/__init__.py +101 -14
- vectordb_bench/backend/clients/aliyun_elasticsearch/aliyun_elasticsearch.py +26 -0
- vectordb_bench/backend/clients/aliyun_elasticsearch/config.py +18 -0
- vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py +345 -0
- vectordb_bench/backend/clients/aliyun_opensearch/config.py +47 -0
- vectordb_bench/backend/clients/alloydb/alloydb.py +58 -80
- vectordb_bench/backend/clients/alloydb/cli.py +52 -35
- vectordb_bench/backend/clients/alloydb/config.py +30 -30
- vectordb_bench/backend/clients/api.py +8 -9
- vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +46 -47
- vectordb_bench/backend/clients/aws_opensearch/cli.py +4 -7
- vectordb_bench/backend/clients/aws_opensearch/config.py +13 -9
- vectordb_bench/backend/clients/aws_opensearch/run.py +69 -59
- vectordb_bench/backend/clients/chroma/chroma.py +38 -36
- vectordb_bench/backend/clients/chroma/config.py +4 -2
- vectordb_bench/backend/clients/elastic_cloud/config.py +5 -5
- vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py +23 -22
- vectordb_bench/backend/clients/memorydb/cli.py +8 -8
- vectordb_bench/backend/clients/memorydb/config.py +2 -2
- vectordb_bench/backend/clients/memorydb/memorydb.py +65 -53
- vectordb_bench/backend/clients/milvus/cli.py +62 -80
- vectordb_bench/backend/clients/milvus/config.py +31 -7
- vectordb_bench/backend/clients/milvus/milvus.py +23 -26
- vectordb_bench/backend/clients/pgdiskann/cli.py +29 -22
- vectordb_bench/backend/clients/pgdiskann/config.py +29 -26
- vectordb_bench/backend/clients/pgdiskann/pgdiskann.py +55 -73
- vectordb_bench/backend/clients/pgvecto_rs/cli.py +9 -11
- vectordb_bench/backend/clients/pgvecto_rs/config.py +8 -14
- vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py +33 -34
- vectordb_bench/backend/clients/pgvector/cli.py +40 -31
- vectordb_bench/backend/clients/pgvector/config.py +63 -73
- vectordb_bench/backend/clients/pgvector/pgvector.py +97 -98
- vectordb_bench/backend/clients/pgvectorscale/cli.py +38 -24
- vectordb_bench/backend/clients/pgvectorscale/config.py +14 -15
- vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py +38 -43
- vectordb_bench/backend/clients/pinecone/config.py +1 -0
- vectordb_bench/backend/clients/pinecone/pinecone.py +14 -21
- vectordb_bench/backend/clients/qdrant_cloud/config.py +11 -10
- vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py +40 -31
- vectordb_bench/backend/clients/redis/cli.py +6 -12
- vectordb_bench/backend/clients/redis/config.py +7 -5
- vectordb_bench/backend/clients/redis/redis.py +94 -58
- vectordb_bench/backend/clients/test/cli.py +1 -2
- vectordb_bench/backend/clients/test/config.py +2 -2
- vectordb_bench/backend/clients/test/test.py +4 -5
- vectordb_bench/backend/clients/weaviate_cloud/cli.py +3 -4
- vectordb_bench/backend/clients/weaviate_cloud/config.py +2 -2
- vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +36 -22
- vectordb_bench/backend/clients/zilliz_cloud/cli.py +14 -11
- vectordb_bench/backend/clients/zilliz_cloud/config.py +2 -4
- vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py +1 -1
- vectordb_bench/backend/data_source.py +30 -18
- vectordb_bench/backend/dataset.py +47 -27
- vectordb_bench/backend/result_collector.py +2 -3
- vectordb_bench/backend/runner/__init__.py +4 -6
- vectordb_bench/backend/runner/mp_runner.py +85 -34
- vectordb_bench/backend/runner/rate_runner.py +51 -23
- vectordb_bench/backend/runner/read_write_runner.py +140 -46
- vectordb_bench/backend/runner/serial_runner.py +99 -50
- vectordb_bench/backend/runner/util.py +4 -19
- vectordb_bench/backend/task_runner.py +95 -74
- vectordb_bench/backend/utils.py +17 -9
- vectordb_bench/base.py +0 -1
- vectordb_bench/cli/cli.py +65 -60
- vectordb_bench/cli/vectordbbench.py +6 -7
- vectordb_bench/frontend/components/check_results/charts.py +8 -19
- vectordb_bench/frontend/components/check_results/data.py +4 -16
- vectordb_bench/frontend/components/check_results/filters.py +8 -16
- vectordb_bench/frontend/components/check_results/nav.py +4 -4
- vectordb_bench/frontend/components/check_results/priceTable.py +1 -3
- vectordb_bench/frontend/components/check_results/stPageConfig.py +2 -1
- vectordb_bench/frontend/components/concurrent/charts.py +12 -12
- vectordb_bench/frontend/components/custom/displayCustomCase.py +17 -11
- vectordb_bench/frontend/components/custom/displaypPrams.py +4 -2
- vectordb_bench/frontend/components/custom/getCustomConfig.py +1 -2
- vectordb_bench/frontend/components/custom/initStyle.py +1 -1
- vectordb_bench/frontend/components/get_results/saveAsImage.py +2 -0
- vectordb_bench/frontend/components/run_test/caseSelector.py +3 -9
- vectordb_bench/frontend/components/run_test/dbConfigSetting.py +1 -4
- vectordb_bench/frontend/components/run_test/dbSelector.py +1 -1
- vectordb_bench/frontend/components/run_test/generateTasks.py +8 -8
- vectordb_bench/frontend/components/run_test/submitTask.py +14 -18
- vectordb_bench/frontend/components/tables/data.py +3 -6
- vectordb_bench/frontend/config/dbCaseConfigs.py +108 -83
- vectordb_bench/frontend/pages/concurrent.py +3 -5
- vectordb_bench/frontend/pages/custom.py +30 -9
- vectordb_bench/frontend/pages/quries_per_dollar.py +3 -3
- vectordb_bench/frontend/pages/run_test.py +3 -7
- vectordb_bench/frontend/utils.py +1 -1
- vectordb_bench/frontend/vdb_benchmark.py +4 -6
- vectordb_bench/interface.py +56 -26
- vectordb_bench/log_util.py +59 -64
- vectordb_bench/metric.py +10 -11
- vectordb_bench/models.py +26 -43
- {vectordb_bench-0.0.18.dist-info → vectordb_bench-0.0.20.dist-info}/METADATA +34 -42
- vectordb_bench-0.0.20.dist-info/RECORD +135 -0
- {vectordb_bench-0.0.18.dist-info → vectordb_bench-0.0.20.dist-info}/WHEEL +1 -1
- vectordb_bench-0.0.18.dist-info/RECORD +0 -131
- {vectordb_bench-0.0.18.dist-info → vectordb_bench-0.0.20.dist-info}/LICENSE +0 -0
- {vectordb_bench-0.0.18.dist-info → vectordb_bench-0.0.20.dist-info}/entry_points.txt +0 -0
- {vectordb_bench-0.0.18.dist-info → vectordb_bench-0.0.20.dist-info}/top_level.txt +0 -0
@@ -32,8 +32,7 @@ def get_custom_configs():
|
|
32
32
|
|
33
33
|
def save_custom_configs(custom_configs: list[CustomDatasetConfig]):
|
34
34
|
with open(config.CUSTOM_CONFIG_DIR, "w") as f:
|
35
|
-
json.dump([custom_config.dict()
|
36
|
-
for custom_config in custom_configs], f, indent=4)
|
35
|
+
json.dump([custom_config.dict() for custom_config in custom_configs], f, indent=4)
|
37
36
|
|
38
37
|
|
39
38
|
def generate_custom_case():
|
@@ -9,10 +9,12 @@ HTML_2_CANVAS_URL = "https://unpkg.com/html2canvas@1.4.1/dist/html2canvas.js"
|
|
9
9
|
def load_unpkg(src: str) -> str:
|
10
10
|
return requests.get(src).text
|
11
11
|
|
12
|
+
|
12
13
|
def getResults(container, pageName="vectordb_bench"):
|
13
14
|
container.subheader("Get results")
|
14
15
|
saveAsImage(container, pageName)
|
15
16
|
|
17
|
+
|
16
18
|
def saveAsImage(container, pageName):
|
17
19
|
html2canvasJS = load_unpkg(HTML_2_CANVAS_URL)
|
18
20
|
container.write()
|
@@ -1,6 +1,4 @@
|
|
1
|
-
|
2
1
|
from vectordb_bench.frontend.config.styles import *
|
3
|
-
from vectordb_bench.backend.cases import CaseType
|
4
2
|
from vectordb_bench.frontend.config.dbCaseConfigs import *
|
5
3
|
from collections import defaultdict
|
6
4
|
|
@@ -23,8 +21,7 @@ def caseSelector(st, activedDbList: list[DB]):
|
|
23
21
|
dbToCaseConfigs = defaultdict(lambda: defaultdict(dict))
|
24
22
|
caseClusters = UI_CASE_CLUSTERS + [get_custom_case_cluter()]
|
25
23
|
for caseCluster in caseClusters:
|
26
|
-
activedCaseList += caseClusterExpander(
|
27
|
-
st, caseCluster, dbToCaseClusterConfigs, activedDbList)
|
24
|
+
activedCaseList += caseClusterExpander(st, caseCluster, dbToCaseClusterConfigs, activedDbList)
|
28
25
|
for db in dbToCaseClusterConfigs:
|
29
26
|
for uiCaseItem in dbToCaseClusterConfigs[db]:
|
30
27
|
for case in uiCaseItem.cases:
|
@@ -40,8 +37,7 @@ def caseClusterExpander(st, caseCluster: UICaseItemCluster, dbToCaseClusterConfi
|
|
40
37
|
if uiCaseItem.isLine:
|
41
38
|
addHorizontalLine(expander)
|
42
39
|
else:
|
43
|
-
activedCases += caseItemCheckbox(expander,
|
44
|
-
dbToCaseClusterConfigs, uiCaseItem, activedDbList)
|
40
|
+
activedCases += caseItemCheckbox(expander, dbToCaseClusterConfigs, uiCaseItem, activedDbList)
|
45
41
|
return activedCases
|
46
42
|
|
47
43
|
|
@@ -53,9 +49,7 @@ def caseItemCheckbox(st, dbToCaseClusterConfigs, uiCaseItem: UICaseItem, actived
|
|
53
49
|
)
|
54
50
|
|
55
51
|
if selected:
|
56
|
-
caseConfigSetting(
|
57
|
-
st.container(), dbToCaseClusterConfigs, uiCaseItem, activedDbList
|
58
|
-
)
|
52
|
+
caseConfigSetting(st.container(), dbToCaseClusterConfigs, uiCaseItem, activedDbList)
|
59
53
|
|
60
54
|
return uiCaseItem.cases if selected else []
|
61
55
|
|
@@ -42,10 +42,7 @@ def dbConfigSettingItem(st, activeDb: DB):
|
|
42
42
|
|
43
43
|
# db config (unique)
|
44
44
|
for key, property in properties.items():
|
45
|
-
if (
|
46
|
-
key not in dbConfigClass.common_short_configs()
|
47
|
-
and key not in dbConfigClass.common_long_configs()
|
48
|
-
):
|
45
|
+
if key not in dbConfigClass.common_short_configs() and key not in dbConfigClass.common_long_configs():
|
49
46
|
column = columns[idx % DB_CONFIG_SETTING_COLUMNS]
|
50
47
|
idx += 1
|
51
48
|
dbConfig[key] = column.text_input(
|
@@ -22,7 +22,7 @@ def dbSelector(st):
|
|
22
22
|
dbIsActived[db] = column.checkbox(db.name)
|
23
23
|
try:
|
24
24
|
column.image(DB_TO_ICON.get(db, ""))
|
25
|
-
except MediaFileStorageError
|
25
|
+
except MediaFileStorageError:
|
26
26
|
column.warning(f"{db.name} image not available")
|
27
27
|
pass
|
28
28
|
activedDbList = [db for db in DB_LIST if dbIsActived[db]]
|
@@ -7,13 +7,13 @@ def generate_tasks(activedDbList: list[DB], dbConfigs, activedCaseList: list[Cas
|
|
7
7
|
for db in activedDbList:
|
8
8
|
for case in activedCaseList:
|
9
9
|
task = TaskConfig(
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
10
|
+
db=db.value,
|
11
|
+
db_config=dbConfigs[db],
|
12
|
+
case_config=case,
|
13
|
+
db_case_config=db.case_config_cls(allCaseConfigs[db][case].get(CaseConfigParamType.IndexType, None))(
|
14
|
+
**{key.value: value for key, value in allCaseConfigs[db][case].items()}
|
15
|
+
),
|
16
|
+
)
|
17
17
|
tasks.append(task)
|
18
|
-
|
18
|
+
|
19
19
|
return tasks
|
@@ -1,6 +1,6 @@
|
|
1
1
|
from datetime import datetime
|
2
|
-
from vectordb_bench.frontend.config
|
3
|
-
from vectordb_bench.interface import
|
2
|
+
from vectordb_bench.frontend.config import styles
|
3
|
+
from vectordb_bench.interface import benchmark_runner
|
4
4
|
|
5
5
|
|
6
6
|
def submitTask(st, tasks, isAllValid):
|
@@ -27,10 +27,8 @@ def submitTask(st, tasks, isAllValid):
|
|
27
27
|
|
28
28
|
def taskLabelInput(st):
|
29
29
|
defaultTaskLabel = datetime.now().strftime("%Y%m%d%H")
|
30
|
-
columns = st.columns(TASK_LABEL_INPUT_COLUMNS)
|
31
|
-
taskLabel = columns[0].text_input(
|
32
|
-
"task_label", defaultTaskLabel, label_visibility="collapsed"
|
33
|
-
)
|
30
|
+
columns = st.columns(styles.TASK_LABEL_INPUT_COLUMNS)
|
31
|
+
taskLabel = columns[0].text_input("task_label", defaultTaskLabel, label_visibility="collapsed")
|
34
32
|
return taskLabel
|
35
33
|
|
36
34
|
|
@@ -46,10 +44,8 @@ def advancedSettings(st):
|
|
46
44
|
)
|
47
45
|
|
48
46
|
container = st.columns([1, 2])
|
49
|
-
k = container[0].number_input("k",min_value=1, value=100, label_visibility="collapsed")
|
50
|
-
container[1].caption(
|
51
|
-
"K value for number of nearest neighbors to search"
|
52
|
-
)
|
47
|
+
k = container[0].number_input("k", min_value=1, value=100, label_visibility="collapsed")
|
48
|
+
container[1].caption("K value for number of nearest neighbors to search")
|
53
49
|
|
54
50
|
return index_already_exists, use_aliyun, k
|
55
51
|
|
@@ -58,20 +54,20 @@ def controlPanel(st, tasks, taskLabel, isAllValid):
|
|
58
54
|
index_already_exists, use_aliyun, k = advancedSettings(st)
|
59
55
|
|
60
56
|
def runHandler():
|
61
|
-
|
57
|
+
benchmark_runner.set_drop_old(not index_already_exists)
|
62
58
|
for task in tasks:
|
63
59
|
task.case_config.k = k
|
64
|
-
|
65
|
-
|
60
|
+
benchmark_runner.set_download_address(use_aliyun)
|
61
|
+
benchmark_runner.run(tasks, taskLabel)
|
66
62
|
|
67
63
|
def stopHandler():
|
68
|
-
|
64
|
+
benchmark_runner.stop_running()
|
69
65
|
|
70
|
-
isRunning =
|
66
|
+
isRunning = benchmark_runner.has_running()
|
71
67
|
|
72
68
|
if isRunning:
|
73
|
-
currentTaskId =
|
74
|
-
tasksCount =
|
69
|
+
currentTaskId = benchmark_runner.get_current_task_id()
|
70
|
+
tasksCount = benchmark_runner.get_tasks_count()
|
75
71
|
text = f":running: Running Task {currentTaskId} / {tasksCount}"
|
76
72
|
st.progress(currentTaskId / tasksCount, text=text)
|
77
73
|
|
@@ -89,7 +85,7 @@ def controlPanel(st, tasks, taskLabel, isAllValid):
|
|
89
85
|
)
|
90
86
|
|
91
87
|
else:
|
92
|
-
errorText =
|
88
|
+
errorText = benchmark_runner.latest_error or ""
|
93
89
|
if len(errorText) > 0:
|
94
90
|
st.error(errorText)
|
95
91
|
disabled = True if len(tasks) == 0 or not isAllValid else False
|
@@ -1,12 +1,11 @@
|
|
1
1
|
from dataclasses import asdict
|
2
|
-
from vectordb_bench.
|
3
|
-
from vectordb_bench.interface import benchMarkRunner
|
2
|
+
from vectordb_bench.interface import benchmark_runner
|
4
3
|
from vectordb_bench.models import CaseResult, ResultLabel
|
5
4
|
import pandas as pd
|
6
5
|
|
7
6
|
|
8
7
|
def getNewResults():
|
9
|
-
allResults =
|
8
|
+
allResults = benchmark_runner.get_results()
|
10
9
|
newResults: list[CaseResult] = []
|
11
10
|
|
12
11
|
for res in allResults:
|
@@ -14,7 +13,6 @@ def getNewResults():
|
|
14
13
|
for result in results:
|
15
14
|
if result.label == ResultLabel.NORMAL:
|
16
15
|
newResults.append(result)
|
17
|
-
|
18
16
|
|
19
17
|
df = pd.DataFrame(formatData(newResults))
|
20
18
|
return df
|
@@ -26,7 +24,6 @@ def formatData(caseResults: list[CaseResult]):
|
|
26
24
|
db = caseResult.task_config.db.value
|
27
25
|
db_label = caseResult.task_config.db_config.db_label
|
28
26
|
case_config = caseResult.task_config.case_config
|
29
|
-
db_case_config = caseResult.task_config.db_case_config
|
30
27
|
case = case_config.case_id.case_cls()
|
31
28
|
filter_rate = case.filter_rate
|
32
29
|
dataset = case.dataset.data.name
|
@@ -41,4 +38,4 @@ def formatData(caseResults: list[CaseResult]):
|
|
41
38
|
**metrics,
|
42
39
|
}
|
43
40
|
)
|
44
|
-
return data
|
41
|
+
return data
|