vectordb-bench 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vectordb_bench/__init__.py +14 -3
- vectordb_bench/backend/assembler.py +2 -2
- vectordb_bench/backend/cases.py +146 -57
- vectordb_bench/backend/clients/__init__.py +6 -1
- vectordb_bench/backend/clients/api.py +23 -11
- vectordb_bench/backend/clients/elastic_cloud/config.py +5 -5
- vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py +11 -9
- vectordb_bench/backend/clients/milvus/config.py +2 -3
- vectordb_bench/backend/clients/milvus/milvus.py +32 -19
- vectordb_bench/backend/clients/pgvector/config.py +49 -0
- vectordb_bench/backend/clients/pgvector/pgvector.py +171 -0
- vectordb_bench/backend/clients/pinecone/config.py +3 -3
- vectordb_bench/backend/clients/pinecone/pinecone.py +19 -13
- vectordb_bench/backend/clients/qdrant_cloud/config.py +23 -6
- vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py +12 -13
- vectordb_bench/backend/clients/weaviate_cloud/config.py +3 -3
- vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +9 -8
- vectordb_bench/backend/clients/zilliz_cloud/config.py +5 -4
- vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py +3 -1
- vectordb_bench/backend/dataset.py +100 -162
- vectordb_bench/backend/result_collector.py +2 -2
- vectordb_bench/backend/runner/mp_runner.py +29 -13
- vectordb_bench/backend/runner/serial_runner.py +98 -36
- vectordb_bench/backend/task_runner.py +43 -48
- vectordb_bench/frontend/components/check_results/charts.py +10 -21
- vectordb_bench/frontend/components/check_results/data.py +31 -15
- vectordb_bench/frontend/components/check_results/expanderStyle.py +37 -0
- vectordb_bench/frontend/components/check_results/filters.py +61 -33
- vectordb_bench/frontend/components/check_results/footer.py +8 -0
- vectordb_bench/frontend/components/check_results/headerIcon.py +8 -4
- vectordb_bench/frontend/components/check_results/nav.py +7 -6
- vectordb_bench/frontend/components/check_results/priceTable.py +3 -2
- vectordb_bench/frontend/components/check_results/stPageConfig.py +18 -0
- vectordb_bench/frontend/components/get_results/saveAsImage.py +50 -0
- vectordb_bench/frontend/components/run_test/autoRefresh.py +1 -1
- vectordb_bench/frontend/components/run_test/caseSelector.py +19 -16
- vectordb_bench/frontend/components/run_test/dbConfigSetting.py +20 -7
- vectordb_bench/frontend/components/run_test/dbSelector.py +5 -5
- vectordb_bench/frontend/components/run_test/hideSidebar.py +4 -6
- vectordb_bench/frontend/components/run_test/submitTask.py +16 -10
- vectordb_bench/frontend/const/dbCaseConfigs.py +291 -0
- vectordb_bench/frontend/const/dbPrices.py +6 -0
- vectordb_bench/frontend/const/styles.py +58 -0
- vectordb_bench/frontend/pages/{qps_with_price.py → quries_per_dollar.py} +24 -17
- vectordb_bench/frontend/pages/run_test.py +17 -11
- vectordb_bench/frontend/vdb_benchmark.py +19 -12
- vectordb_bench/metric.py +19 -10
- vectordb_bench/models.py +14 -40
- vectordb_bench/results/dbPrices.json +32 -0
- vectordb_bench/results/getLeaderboardData.py +52 -0
- vectordb_bench/results/leaderboard.json +1 -0
- vectordb_bench/results/{result_20230609_standard.json → result_20230705_standard.json} +1910 -897
- {vectordb_bench-0.0.1.dist-info → vectordb_bench-0.0.3.dist-info}/METADATA +107 -27
- vectordb_bench-0.0.3.dist-info/RECORD +67 -0
- vectordb_bench/frontend/const.py +0 -391
- vectordb_bench-0.0.1.dist-info/RECORD +0 -56
- {vectordb_bench-0.0.1.dist-info → vectordb_bench-0.0.3.dist-info}/LICENSE +0 -0
- {vectordb_bench-0.0.1.dist-info → vectordb_bench-0.0.3.dist-info}/WHEEL +0 -0
- {vectordb_bench-0.0.1.dist-info → vectordb_bench-0.0.3.dist-info}/entry_points.txt +0 -0
- {vectordb_bench-0.0.1.dist-info → vectordb_bench-0.0.3.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,6 @@
|
|
1
|
-
from vectordb_bench.frontend.const import *
|
1
|
+
from vectordb_bench.frontend.const.styles import *
|
2
|
+
from vectordb_bench.backend.cases import CaseType
|
3
|
+
from vectordb_bench.frontend.const.dbCaseConfigs import *
|
2
4
|
|
3
5
|
|
4
6
|
def caseSelector(st, activedDbList):
|
@@ -12,34 +14,35 @@ def caseSelector(st, activedDbList):
|
|
12
14
|
unsafe_allow_html=True,
|
13
15
|
)
|
14
16
|
|
15
|
-
caseIsActived = {case
|
16
|
-
allCaseConfigs = {db: {case
|
17
|
-
for
|
18
|
-
|
19
|
-
caseIsActived[case["name"]] = caseItem(
|
20
|
-
caseItemContainer, allCaseConfigs, case, activedDbList
|
21
|
-
)
|
22
|
-
if case.get("divider"):
|
17
|
+
caseIsActived = {case: False for case in CASE_LIST}
|
18
|
+
allCaseConfigs = {db: {case: {} for case in CASE_LIST} for db in DB_LIST}
|
19
|
+
for caseOrDivider in CASE_LIST_WITH_DIVIDER:
|
20
|
+
if caseOrDivider == DIVIDER:
|
23
21
|
caseItemContainer.markdown(
|
24
22
|
"<div style='border: 1px solid #cccccc60; margin-bottom: 24px;'></div>",
|
25
23
|
unsafe_allow_html=True,
|
26
24
|
)
|
27
|
-
|
28
|
-
|
29
|
-
|
25
|
+
else:
|
26
|
+
case = caseOrDivider
|
27
|
+
caseItemContainer = st.container()
|
28
|
+
caseIsActived[case] = caseItem(
|
29
|
+
caseItemContainer, allCaseConfigs, case, activedDbList
|
30
|
+
)
|
31
|
+
activedCaseList = [case for case in CASE_LIST if caseIsActived[case]]
|
30
32
|
return activedCaseList, allCaseConfigs
|
31
33
|
|
32
|
-
|
33
|
-
|
34
|
+
|
35
|
+
def caseItem(st, allCaseConfigs, case: CaseType, activedDbList):
|
36
|
+
selected = st.checkbox(case.case_name)
|
34
37
|
st.markdown(
|
35
|
-
f"<div style='color: #1D2939; margin: -8px 0 20px {CHECKBOX_INDENT}px; font-size: 14px;'>{case
|
38
|
+
f"<div style='color: #1D2939; margin: -8px 0 20px {CHECKBOX_INDENT}px; font-size: 14px;'>{case.case_description}</div>",
|
36
39
|
unsafe_allow_html=True,
|
37
40
|
)
|
38
41
|
|
39
42
|
if selected:
|
40
43
|
caseConfigSettingContainer = st.container()
|
41
44
|
caseConfigSetting(
|
42
|
-
caseConfigSettingContainer, allCaseConfigs, case
|
45
|
+
caseConfigSettingContainer, allCaseConfigs, case, activedDbList
|
43
46
|
)
|
44
47
|
|
45
48
|
return selected
|
@@ -1,4 +1,5 @@
|
|
1
|
-
from
|
1
|
+
from pydantic import ValidationError
|
2
|
+
from vectordb_bench.frontend.const.styles import *
|
2
3
|
from vectordb_bench.frontend.utils import inputIsPassword
|
3
4
|
|
4
5
|
|
@@ -10,12 +11,25 @@ def dbConfigSettings(st, activedDbList):
|
|
10
11
|
expander = st.expander("Configurations for the selected databases", True)
|
11
12
|
|
12
13
|
dbConfigs = {}
|
14
|
+
isAllValid = True
|
13
15
|
for activeDb in activedDbList:
|
14
16
|
dbConfigSettingItemContainer = expander.container()
|
15
17
|
dbConfig = dbConfigSettingItem(dbConfigSettingItemContainer, activeDb)
|
16
|
-
|
17
|
-
|
18
|
-
|
18
|
+
try:
|
19
|
+
dbConfigs[activeDb] = activeDb.init_cls.config_cls()(**dbConfig)
|
20
|
+
except ValidationError as e:
|
21
|
+
isAllValid = False
|
22
|
+
errTexts = []
|
23
|
+
for err in e.raw_errors:
|
24
|
+
errLocs = err.loc_tuple()
|
25
|
+
errInfo = err.exc
|
26
|
+
errText = f"{', '.join(errLocs)} - {errInfo}"
|
27
|
+
errTexts.append(errText)
|
28
|
+
|
29
|
+
dbConfigSettingItemContainer.error(f"{'; '.join(errTexts)}")
|
30
|
+
|
31
|
+
return dbConfigs, isAllValid
|
32
|
+
|
19
33
|
|
20
34
|
def dbConfigSettingItem(st, activeDb):
|
21
35
|
st.markdown(
|
@@ -39,9 +53,8 @@ def dbConfigSettingItem(st, activeDb):
|
|
39
53
|
value=value.get("default", ""),
|
40
54
|
type="password" if inputIsPassword(key) else "default",
|
41
55
|
)
|
42
|
-
return
|
56
|
+
return dbConfig
|
43
57
|
|
44
58
|
|
45
59
|
def moveDBLabelToLast(propertiesItems):
|
46
|
-
propertiesItems.sort(key=lambda x: 1 if x[0] ==
|
47
|
-
|
60
|
+
propertiesItems.sort(key=lambda x: 1 if x[0] == "db_label" else 0)
|
@@ -1,5 +1,5 @@
|
|
1
|
-
|
2
|
-
from vectordb_bench.frontend.const import
|
1
|
+
from vectordb_bench.frontend.const.styles import *
|
2
|
+
from vectordb_bench.frontend.const.dbCaseConfigs import DB_LIST
|
3
3
|
|
4
4
|
|
5
5
|
def dbSelector(st):
|
@@ -12,10 +12,10 @@ def dbSelector(st):
|
|
12
12
|
"<div style='color: #647489; margin-bottom: 24px; margin-top: -12px;'>Choose at least one case you want to run the test for. </div>",
|
13
13
|
unsafe_allow_html=True,
|
14
14
|
)
|
15
|
-
|
15
|
+
|
16
16
|
dbContainerColumns = st.columns(DB_SELECTOR_COLUMNS, gap="small")
|
17
17
|
dbIsActived = {db: False for db in DB_LIST}
|
18
|
-
|
18
|
+
|
19
19
|
# style - image; column gap; checkbox font;
|
20
20
|
st.markdown(
|
21
21
|
"""
|
@@ -32,5 +32,5 @@ def dbSelector(st):
|
|
32
32
|
dbIsActived[db] = column.checkbox(db.name)
|
33
33
|
column.image(DB_TO_ICON.get(db, ""))
|
34
34
|
activedDbList = [db for db in DB_LIST if dbIsActived[db]]
|
35
|
-
|
35
|
+
|
36
36
|
return activedDbList
|
@@ -1,10 +1,8 @@
|
|
1
1
|
def hideSidebar(st):
|
2
2
|
st.markdown(
|
3
|
-
"<style>
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
st.markdown(
|
8
|
-
"<style> .block-container { max-width: 1000px; } </style>",
|
3
|
+
"""<style>
|
4
|
+
div[data-testid='collapsedControl'] {display: none;}
|
5
|
+
.block-container { max-width: 1000px; }
|
6
|
+
</style>""",
|
9
7
|
unsafe_allow_html=True,
|
10
8
|
)
|
@@ -1,9 +1,9 @@
|
|
1
1
|
from datetime import datetime
|
2
|
-
from vectordb_bench.frontend.const import *
|
2
|
+
from vectordb_bench.frontend.const.styles import *
|
3
3
|
from vectordb_bench.interface import benchMarkRunner
|
4
4
|
|
5
5
|
|
6
|
-
def submitTask(st, tasks):
|
6
|
+
def submitTask(st, tasks, isAllValid):
|
7
7
|
st.markdown(
|
8
8
|
"<div style='height: 24px;'></div>",
|
9
9
|
unsafe_allow_html=True,
|
@@ -15,34 +15,36 @@ def submitTask(st, tasks):
|
|
15
15
|
)
|
16
16
|
|
17
17
|
taskLabel = taskLabelInput(st)
|
18
|
-
|
18
|
+
|
19
19
|
st.markdown(
|
20
20
|
"<div style='height: 24px;'></div>",
|
21
21
|
unsafe_allow_html=True,
|
22
22
|
)
|
23
23
|
|
24
24
|
controlPanelContainer = st.container()
|
25
|
-
controlPanel(controlPanelContainer, tasks, taskLabel)
|
25
|
+
controlPanel(controlPanelContainer, tasks, taskLabel, isAllValid)
|
26
26
|
|
27
27
|
|
28
28
|
def taskLabelInput(st):
|
29
29
|
defaultTaskLabel = datetime.now().strftime("%Y%m%d%H")
|
30
30
|
columns = st.columns(TASK_LABEL_INPUT_COLUMNS)
|
31
|
-
taskLabel = columns[0].text_input(
|
31
|
+
taskLabel = columns[0].text_input(
|
32
|
+
"task_label", defaultTaskLabel, label_visibility="collapsed"
|
33
|
+
)
|
32
34
|
return taskLabel
|
33
35
|
|
34
36
|
|
35
|
-
def controlPanel(st, tasks, taskLabel):
|
37
|
+
def controlPanel(st, tasks, taskLabel, isAllValid):
|
36
38
|
isRunning = benchMarkRunner.has_running()
|
37
39
|
runHandler = lambda: benchMarkRunner.run(tasks, taskLabel)
|
38
40
|
stopHandler = lambda: benchMarkRunner.stop_running()
|
39
|
-
|
41
|
+
|
40
42
|
if isRunning:
|
41
43
|
currentTaskId = benchMarkRunner.get_current_task_id()
|
42
44
|
tasksCount = benchMarkRunner.get_tasks_count()
|
43
45
|
text = f":running: Running Task {currentTaskId} / {tasksCount}"
|
44
46
|
st.progress(currentTaskId / tasksCount, text=text)
|
45
|
-
|
47
|
+
|
46
48
|
columns = st.columns(6)
|
47
49
|
columns[0].button(
|
48
50
|
"Run Your Test",
|
@@ -55,12 +57,16 @@ def controlPanel(st, tasks, taskLabel):
|
|
55
57
|
on_click=stopHandler,
|
56
58
|
type="primary",
|
57
59
|
)
|
58
|
-
|
60
|
+
|
59
61
|
else:
|
60
62
|
errorText = benchMarkRunner.latest_error or ""
|
61
63
|
if len(errorText) > 0:
|
62
64
|
st.error(errorText)
|
63
|
-
disabled = True if len(tasks) == 0 else False
|
65
|
+
disabled = True if len(tasks) == 0 or not isAllValid else False
|
66
|
+
if not isAllValid:
|
67
|
+
st.error("Make sure all config is valid.")
|
68
|
+
elif len(tasks) == 0:
|
69
|
+
st.warning("No tests to run.")
|
64
70
|
st.button(
|
65
71
|
"Run Your Test",
|
66
72
|
disabled=disabled,
|
@@ -0,0 +1,291 @@
|
|
1
|
+
from enum import IntEnum
|
2
|
+
import typing
|
3
|
+
from pydantic import BaseModel
|
4
|
+
from vectordb_bench.backend.cases import CaseType
|
5
|
+
from vectordb_bench.backend.clients import DB
|
6
|
+
from vectordb_bench.backend.clients.api import IndexType
|
7
|
+
|
8
|
+
from vectordb_bench.models import CaseConfigParamType
|
9
|
+
|
10
|
+
MAX_STREAMLIT_INT = (1 << 53) - 1
|
11
|
+
|
12
|
+
DB_LIST = [d for d in DB]
|
13
|
+
|
14
|
+
DIVIDER = "DIVIDER"
|
15
|
+
CASE_LIST_WITH_DIVIDER = [
|
16
|
+
CaseType.Performance100M,
|
17
|
+
CaseType.Performance10M,
|
18
|
+
CaseType.Performance1M,
|
19
|
+
DIVIDER,
|
20
|
+
CaseType.Performance10M1P,
|
21
|
+
CaseType.Performance1M1P,
|
22
|
+
DIVIDER,
|
23
|
+
CaseType.Performance10M99P,
|
24
|
+
CaseType.Performance1M99P,
|
25
|
+
DIVIDER,
|
26
|
+
CaseType.CapacityDim960,
|
27
|
+
CaseType.CapacityDim128,
|
28
|
+
]
|
29
|
+
|
30
|
+
CASE_LIST = [item for item in CASE_LIST_WITH_DIVIDER if isinstance(item, CaseType)]
|
31
|
+
|
32
|
+
|
33
|
+
class InputType(IntEnum):
|
34
|
+
Text = 20001
|
35
|
+
Number = 20002
|
36
|
+
Option = 20003
|
37
|
+
|
38
|
+
|
39
|
+
class CaseConfigInput(BaseModel):
|
40
|
+
label: CaseConfigParamType
|
41
|
+
inputType: InputType = InputType.Text
|
42
|
+
inputConfig: dict = {}
|
43
|
+
# todo type should be a function
|
44
|
+
isDisplayed: typing.Any = lambda x: True
|
45
|
+
|
46
|
+
|
47
|
+
CaseConfigParamInput_IndexType = CaseConfigInput(
|
48
|
+
label=CaseConfigParamType.IndexType,
|
49
|
+
inputType=InputType.Option,
|
50
|
+
inputConfig={
|
51
|
+
"options": [
|
52
|
+
IndexType.HNSW.value,
|
53
|
+
IndexType.IVFFlat.value,
|
54
|
+
IndexType.DISKANN.value,
|
55
|
+
IndexType.Flat.value,
|
56
|
+
IndexType.AUTOINDEX.value,
|
57
|
+
],
|
58
|
+
},
|
59
|
+
)
|
60
|
+
|
61
|
+
CaseConfigParamInput_M = CaseConfigInput(
|
62
|
+
label=CaseConfigParamType.M,
|
63
|
+
inputType=InputType.Number,
|
64
|
+
inputConfig={
|
65
|
+
"min": 4,
|
66
|
+
"max": 64,
|
67
|
+
"value": 30,
|
68
|
+
},
|
69
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
70
|
+
== IndexType.HNSW.value,
|
71
|
+
)
|
72
|
+
|
73
|
+
CaseConfigParamInput_EFConstruction_Milvus = CaseConfigInput(
|
74
|
+
label=CaseConfigParamType.EFConstruction,
|
75
|
+
inputType=InputType.Number,
|
76
|
+
inputConfig={
|
77
|
+
"min": 8,
|
78
|
+
"max": 512,
|
79
|
+
"value": 360,
|
80
|
+
},
|
81
|
+
isDisplayed=lambda config: config[CaseConfigParamType.IndexType]
|
82
|
+
== IndexType.HNSW.value,
|
83
|
+
)
|
84
|
+
|
85
|
+
CaseConfigParamInput_EFConstruction_Weaviate = CaseConfigInput(
|
86
|
+
label=CaseConfigParamType.EFConstruction,
|
87
|
+
inputType=InputType.Number,
|
88
|
+
inputConfig={
|
89
|
+
"min": 8,
|
90
|
+
"max": 512,
|
91
|
+
"value": 128,
|
92
|
+
},
|
93
|
+
)
|
94
|
+
|
95
|
+
CaseConfigParamInput_EFConstruction_ES = CaseConfigInput(
|
96
|
+
label=CaseConfigParamType.EFConstruction,
|
97
|
+
inputType=InputType.Number,
|
98
|
+
inputConfig={
|
99
|
+
"min": 8,
|
100
|
+
"max": 512,
|
101
|
+
"value": 360,
|
102
|
+
},
|
103
|
+
)
|
104
|
+
|
105
|
+
CaseConfigParamInput_M_ES = CaseConfigInput(
|
106
|
+
label=CaseConfigParamType.M,
|
107
|
+
inputType=InputType.Number,
|
108
|
+
inputConfig={
|
109
|
+
"min": 4,
|
110
|
+
"max": 64,
|
111
|
+
"value": 30,
|
112
|
+
},
|
113
|
+
)
|
114
|
+
|
115
|
+
CaseConfigParamInput_NumCandidates_ES = CaseConfigInput(
|
116
|
+
label=CaseConfigParamType.numCandidates,
|
117
|
+
inputType=InputType.Number,
|
118
|
+
inputConfig={
|
119
|
+
"min": 1,
|
120
|
+
"max": 10000,
|
121
|
+
"value": 100,
|
122
|
+
},
|
123
|
+
)
|
124
|
+
|
125
|
+
CaseConfigParamInput_EF_Milvus = CaseConfigInput(
|
126
|
+
label=CaseConfigParamType.EF,
|
127
|
+
inputType=InputType.Number,
|
128
|
+
inputConfig={
|
129
|
+
"min": 100,
|
130
|
+
"max": MAX_STREAMLIT_INT,
|
131
|
+
"value": 100,
|
132
|
+
},
|
133
|
+
isDisplayed=lambda config: config[CaseConfigParamType.IndexType]
|
134
|
+
== IndexType.HNSW.value,
|
135
|
+
)
|
136
|
+
|
137
|
+
CaseConfigParamInput_EF_Weaviate = CaseConfigInput(
|
138
|
+
label=CaseConfigParamType.EF,
|
139
|
+
inputType=InputType.Number,
|
140
|
+
inputConfig={
|
141
|
+
"min": -1,
|
142
|
+
"max": MAX_STREAMLIT_INT,
|
143
|
+
"value": -1,
|
144
|
+
},
|
145
|
+
)
|
146
|
+
|
147
|
+
CaseConfigParamInput_MaxConnections = CaseConfigInput(
|
148
|
+
label=CaseConfigParamType.MaxConnections,
|
149
|
+
inputType=InputType.Number,
|
150
|
+
inputConfig={"min": 1, "max": MAX_STREAMLIT_INT, "value": 64},
|
151
|
+
)
|
152
|
+
|
153
|
+
CaseConfigParamInput_SearchList = CaseConfigInput(
|
154
|
+
label=CaseConfigParamType.SearchList,
|
155
|
+
inputType=InputType.Number,
|
156
|
+
inputConfig={
|
157
|
+
"min": 100,
|
158
|
+
"max": MAX_STREAMLIT_INT,
|
159
|
+
"value": 100,
|
160
|
+
},
|
161
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
162
|
+
== IndexType.DISKANN.value,
|
163
|
+
)
|
164
|
+
|
165
|
+
CaseConfigParamInput_Nlist = CaseConfigInput(
|
166
|
+
label=CaseConfigParamType.Nlist,
|
167
|
+
inputType=InputType.Number,
|
168
|
+
inputConfig={
|
169
|
+
"min": 1,
|
170
|
+
"max": 65536,
|
171
|
+
"value": 1000,
|
172
|
+
},
|
173
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
174
|
+
== IndexType.IVFFlat.value,
|
175
|
+
)
|
176
|
+
|
177
|
+
CaseConfigParamInput_Nprobe = CaseConfigInput(
|
178
|
+
label=CaseConfigParamType.Nprobe,
|
179
|
+
inputType=InputType.Number,
|
180
|
+
inputConfig={
|
181
|
+
"min": 1,
|
182
|
+
"max": 65536,
|
183
|
+
"value": 10,
|
184
|
+
},
|
185
|
+
isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
|
186
|
+
== IndexType.IVFFlat.value,
|
187
|
+
)
|
188
|
+
|
189
|
+
CaseConfigParamInput_Lists = CaseConfigInput(
|
190
|
+
label=CaseConfigParamType.lists,
|
191
|
+
inputType=InputType.Number,
|
192
|
+
inputConfig={
|
193
|
+
"min": 1,
|
194
|
+
"max": 65536,
|
195
|
+
"value": 10,
|
196
|
+
},
|
197
|
+
)
|
198
|
+
|
199
|
+
CaseConfigParamInput_Probes = CaseConfigInput(
|
200
|
+
label=CaseConfigParamType.probes,
|
201
|
+
inputType=InputType.Number,
|
202
|
+
inputConfig={
|
203
|
+
"min": 1,
|
204
|
+
"max": 65536,
|
205
|
+
"value": 1,
|
206
|
+
},
|
207
|
+
)
|
208
|
+
|
209
|
+
|
210
|
+
MilvusLoadConfig = [
|
211
|
+
CaseConfigParamInput_IndexType,
|
212
|
+
CaseConfigParamInput_M,
|
213
|
+
CaseConfigParamInput_EFConstruction_Milvus,
|
214
|
+
CaseConfigParamInput_Nlist,
|
215
|
+
]
|
216
|
+
MilvusPerformanceConfig = [
|
217
|
+
CaseConfigParamInput_IndexType,
|
218
|
+
CaseConfigParamInput_M,
|
219
|
+
CaseConfigParamInput_EFConstruction_Milvus,
|
220
|
+
CaseConfigParamInput_EF_Milvus,
|
221
|
+
CaseConfigParamInput_SearchList,
|
222
|
+
CaseConfigParamInput_Nlist,
|
223
|
+
CaseConfigParamInput_Nprobe,
|
224
|
+
]
|
225
|
+
|
226
|
+
WeaviateLoadConfig = [
|
227
|
+
CaseConfigParamInput_MaxConnections,
|
228
|
+
CaseConfigParamInput_EFConstruction_Weaviate,
|
229
|
+
]
|
230
|
+
WeaviatePerformanceConfig = [
|
231
|
+
CaseConfigParamInput_MaxConnections,
|
232
|
+
CaseConfigParamInput_EFConstruction_Weaviate,
|
233
|
+
CaseConfigParamInput_EF_Weaviate,
|
234
|
+
]
|
235
|
+
|
236
|
+
ESLoadingConfig = [CaseConfigParamInput_EFConstruction_ES, CaseConfigParamInput_M_ES]
|
237
|
+
ESPerformanceConfig = [
|
238
|
+
CaseConfigParamInput_EFConstruction_ES,
|
239
|
+
CaseConfigParamInput_M_ES,
|
240
|
+
CaseConfigParamInput_NumCandidates_ES,
|
241
|
+
]
|
242
|
+
|
243
|
+
PgVectorLoadingConfig = [CaseConfigParamInput_Lists]
|
244
|
+
PgVectorPerformanceConfig = [CaseConfigParamInput_Lists, CaseConfigParamInput_Probes]
|
245
|
+
|
246
|
+
CASE_CONFIG_MAP = {
|
247
|
+
DB.Milvus: {
|
248
|
+
CaseType.CapacityDim960: MilvusLoadConfig,
|
249
|
+
CaseType.CapacityDim128: MilvusLoadConfig,
|
250
|
+
CaseType.Performance100M: MilvusPerformanceConfig,
|
251
|
+
CaseType.Performance10M: MilvusPerformanceConfig,
|
252
|
+
CaseType.Performance1M: MilvusPerformanceConfig,
|
253
|
+
CaseType.Performance10M1P: MilvusPerformanceConfig,
|
254
|
+
CaseType.Performance1M1P: MilvusPerformanceConfig,
|
255
|
+
CaseType.Performance10M99P: MilvusPerformanceConfig,
|
256
|
+
CaseType.Performance1M99P: MilvusPerformanceConfig,
|
257
|
+
},
|
258
|
+
DB.WeaviateCloud: {
|
259
|
+
CaseType.CapacityDim960: WeaviateLoadConfig,
|
260
|
+
CaseType.CapacityDim128: WeaviateLoadConfig,
|
261
|
+
CaseType.Performance100M: WeaviatePerformanceConfig,
|
262
|
+
CaseType.Performance10M: WeaviatePerformanceConfig,
|
263
|
+
CaseType.Performance1M: WeaviatePerformanceConfig,
|
264
|
+
CaseType.Performance10M1P: WeaviatePerformanceConfig,
|
265
|
+
CaseType.Performance1M1P: WeaviatePerformanceConfig,
|
266
|
+
CaseType.Performance10M99P: WeaviatePerformanceConfig,
|
267
|
+
CaseType.Performance1M99P: WeaviatePerformanceConfig,
|
268
|
+
},
|
269
|
+
DB.ElasticCloud: {
|
270
|
+
CaseType.CapacityDim960: ESLoadingConfig,
|
271
|
+
CaseType.CapacityDim128: ESLoadingConfig,
|
272
|
+
CaseType.Performance100M: ESPerformanceConfig,
|
273
|
+
CaseType.Performance10M: ESPerformanceConfig,
|
274
|
+
CaseType.Performance1M: ESPerformanceConfig,
|
275
|
+
CaseType.Performance10M1P: ESPerformanceConfig,
|
276
|
+
CaseType.Performance1M1P: ESPerformanceConfig,
|
277
|
+
CaseType.Performance10M99P: ESPerformanceConfig,
|
278
|
+
CaseType.Performance1M99P: ESPerformanceConfig,
|
279
|
+
},
|
280
|
+
DB.PgVector: {
|
281
|
+
CaseType.CapacityDim960: PgVectorLoadingConfig,
|
282
|
+
CaseType.CapacityDim128: PgVectorLoadingConfig,
|
283
|
+
CaseType.Performance100M: PgVectorPerformanceConfig,
|
284
|
+
CaseType.Performance10M: PgVectorPerformanceConfig,
|
285
|
+
CaseType.Performance1M: PgVectorPerformanceConfig,
|
286
|
+
CaseType.Performance10M1P: PgVectorPerformanceConfig,
|
287
|
+
CaseType.Performance1M1P: PgVectorPerformanceConfig,
|
288
|
+
CaseType.Performance10M99P: PgVectorPerformanceConfig,
|
289
|
+
CaseType.Performance1M99P: PgVectorPerformanceConfig,
|
290
|
+
},
|
291
|
+
}
|
@@ -0,0 +1,58 @@
|
|
1
|
+
from vectordb_bench.models import DB
|
2
|
+
|
3
|
+
# style const
|
4
|
+
DB_SELECTOR_COLUMNS = 6
|
5
|
+
DB_CONFIG_SETTING_COLUMNS = 3
|
6
|
+
CASE_CONFIG_SETTING_COLUMNS = 4
|
7
|
+
CHECKBOX_INDENT = 30
|
8
|
+
TASK_LABEL_INPUT_COLUMNS = 2
|
9
|
+
CHECKBOX_MAX_COLUMNS = 4
|
10
|
+
DB_CONFIG_INPUT_MAX_COLUMNS = 2
|
11
|
+
CASE_CONFIG_INPUT_MAX_COLUMNS = 3
|
12
|
+
DB_CONFIG_INPUT_WIDTH_RADIO = 2
|
13
|
+
CASE_CONFIG_INPUT_WIDTH_RADIO = 0.98
|
14
|
+
CASE_INTRO_RATIO = 3
|
15
|
+
SIDEBAR_CONTROL_COLUMNS = 3
|
16
|
+
LEGEND_RECT_WIDTH = 24
|
17
|
+
LEGEND_RECT_HEIGHT = 16
|
18
|
+
LEGEND_TEXT_FONT_SIZE = 14
|
19
|
+
|
20
|
+
PATTERN_SHAPES = ["", "+", "\\", "x", ".", "|", "/", "-"]
|
21
|
+
|
22
|
+
|
23
|
+
def getPatternShape(i):
|
24
|
+
return PATTERN_SHAPES[i % len(PATTERN_SHAPES)]
|
25
|
+
|
26
|
+
|
27
|
+
# run_test page auto-refresh config
|
28
|
+
MAX_AUTO_REFRESH_COUNT = 999999
|
29
|
+
MAX_AUTO_REFRESH_INTERVAL = 5000 # 5s
|
30
|
+
|
31
|
+
PAGE_TITLE = "VectorDB Benchmark"
|
32
|
+
FAVICON = "https://assets.zilliz.com/favicon_f7f922fe27.png"
|
33
|
+
HEADER_ICON = "https://assets.zilliz.com/vdb_benchmark_db790b5387.png"
|
34
|
+
|
35
|
+
# RedisCloud icon: https://assets.zilliz.com/Redis_Cloud_74b8bfef39.png
|
36
|
+
# Elasticsearch icon: https://assets.zilliz.com/elasticsearch_beffeadc29.png
|
37
|
+
# Chroma icon: https://assets.zilliz.com/chroma_ceb3f06ed7.png
|
38
|
+
DB_TO_ICON = {
|
39
|
+
DB.Milvus: "https://assets.zilliz.com/milvus_c30b0d1994.png",
|
40
|
+
DB.ZillizCloud: "https://assets.zilliz.com/zilliz_5f4cc9b050.png",
|
41
|
+
DB.ElasticCloud: "https://assets.zilliz.com/Elatic_Cloud_dad8d6a3a3.png",
|
42
|
+
DB.Pinecone: "https://assets.zilliz.com/pinecone_94d8154979.png",
|
43
|
+
DB.QdrantCloud: "https://assets.zilliz.com/qdrant_b691674fcd.png",
|
44
|
+
DB.WeaviateCloud: "https://assets.zilliz.com/weaviate_4f6f171ebe.png",
|
45
|
+
DB.PgVector: "https://assets.zilliz.com/PG_Vector_d464f2ef5f.png",
|
46
|
+
}
|
47
|
+
|
48
|
+
# RedisCloud color: #0D6EFD
|
49
|
+
# Chroma color: #FFC107
|
50
|
+
COLOR_MAP = {
|
51
|
+
DB.Milvus.value: "#0DCAF0",
|
52
|
+
DB.ZillizCloud.value: "#0D6EFD",
|
53
|
+
DB.ElasticCloud.value: "#04D6C8",
|
54
|
+
DB.Pinecone.value: "#6610F2",
|
55
|
+
DB.QdrantCloud.value: "#D91AD9",
|
56
|
+
DB.WeaviateCloud.value: "#20C997",
|
57
|
+
DB.PgVector.value: "#4C779A",
|
58
|
+
}
|
@@ -1,32 +1,32 @@
|
|
1
1
|
import streamlit as st
|
2
|
+
from vectordb_bench.frontend.components.check_results.footer import footer
|
3
|
+
from vectordb_bench.frontend.components.check_results.expanderStyle import initMainExpanderStyle
|
2
4
|
from vectordb_bench.frontend.components.check_results.priceTable import priceTable
|
3
|
-
from vectordb_bench.frontend.
|
5
|
+
from vectordb_bench.frontend.components.check_results.stPageConfig import initResultsPageConfig
|
4
6
|
from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
|
5
7
|
from vectordb_bench.frontend.components.check_results.nav import NavToResults, NavToRunTest
|
6
8
|
from vectordb_bench.frontend.components.check_results.charts import drawMetricChart
|
7
9
|
from vectordb_bench.frontend.components.check_results.filters import getshownData
|
10
|
+
from vectordb_bench.frontend.components.get_results.saveAsImage import getResults
|
11
|
+
from vectordb_bench.frontend.const.styles import *
|
8
12
|
from vectordb_bench.interface import benchMarkRunner
|
13
|
+
from vectordb_bench.metric import QURIES_PER_DOLLAR_METRIC
|
9
14
|
|
10
15
|
|
11
16
|
def main():
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
# layout="wide",
|
16
|
-
# initial_sidebar_state="collapsed",
|
17
|
-
)
|
18
|
-
|
17
|
+
# set page config
|
18
|
+
initResultsPageConfig(st)
|
19
|
+
|
19
20
|
# header
|
20
21
|
drawHeaderIcon(st)
|
21
22
|
|
22
23
|
allResults = benchMarkRunner.get_results()
|
23
24
|
|
24
|
-
st.title("Vector DB Benchmark (
|
25
|
-
st.subheader("Price List")
|
25
|
+
st.title("Vector DB Benchmark (QP$)")
|
26
26
|
|
27
27
|
# results selector
|
28
28
|
resultSelectorContainer = st.sidebar.container()
|
29
|
-
shownData,
|
29
|
+
shownData, _, showCases = getshownData(allResults, resultSelectorContainer)
|
30
30
|
|
31
31
|
resultSelectorContainer.divider()
|
32
32
|
|
@@ -34,27 +34,34 @@ def main():
|
|
34
34
|
navContainer = st.sidebar.container()
|
35
35
|
NavToRunTest(navContainer)
|
36
36
|
NavToResults(navContainer)
|
37
|
-
|
37
|
+
|
38
|
+
# save or share
|
39
|
+
resultesContainer = st.sidebar.container()
|
40
|
+
getResults(resultesContainer, "vectordb_bench_qp$")
|
41
|
+
|
38
42
|
# price table
|
43
|
+
initMainExpanderStyle(st)
|
39
44
|
priceTableContainer = st.container()
|
40
45
|
priceMap = priceTable(priceTableContainer, shownData)
|
41
46
|
|
42
47
|
# charts
|
43
48
|
for case in showCases:
|
44
|
-
|
45
|
-
data = [data for data in shownData if data["case"] == case]
|
49
|
+
data = [data for data in shownData if data["case_name"] == case.name]
|
46
50
|
dataWithMetric = []
|
47
|
-
metric =
|
51
|
+
metric = QURIES_PER_DOLLAR_METRIC
|
48
52
|
for d in data:
|
49
53
|
qps = d.get("qps", 0)
|
50
54
|
price = priceMap.get(d["db"], {}).get(d["db_label"], 0)
|
51
55
|
if qps > 0 and price > 0:
|
52
|
-
d[metric] = d["qps"] / price
|
56
|
+
d[metric] = d["qps"] / price * 3.6
|
53
57
|
dataWithMetric.append(d)
|
54
58
|
if len(dataWithMetric) > 0:
|
55
|
-
chartContainer.
|
59
|
+
chartContainer = st.expander(case.name, True)
|
56
60
|
drawMetricChart(data, metric, chartContainer)
|
57
61
|
|
62
|
+
# footer
|
63
|
+
footer(st.container())
|
64
|
+
|
58
65
|
|
59
66
|
if __name__ == "__main__":
|
60
67
|
main()
|