vectordb-bench 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. vectordb_bench/__init__.py +14 -3
  2. vectordb_bench/backend/assembler.py +2 -2
  3. vectordb_bench/backend/cases.py +146 -57
  4. vectordb_bench/backend/clients/__init__.py +6 -1
  5. vectordb_bench/backend/clients/api.py +23 -11
  6. vectordb_bench/backend/clients/elastic_cloud/config.py +5 -5
  7. vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py +11 -9
  8. vectordb_bench/backend/clients/milvus/config.py +2 -3
  9. vectordb_bench/backend/clients/milvus/milvus.py +32 -19
  10. vectordb_bench/backend/clients/pgvector/config.py +49 -0
  11. vectordb_bench/backend/clients/pgvector/pgvector.py +171 -0
  12. vectordb_bench/backend/clients/pinecone/config.py +3 -3
  13. vectordb_bench/backend/clients/pinecone/pinecone.py +19 -13
  14. vectordb_bench/backend/clients/qdrant_cloud/config.py +23 -6
  15. vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py +12 -13
  16. vectordb_bench/backend/clients/weaviate_cloud/config.py +3 -3
  17. vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +9 -8
  18. vectordb_bench/backend/clients/zilliz_cloud/config.py +5 -4
  19. vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py +3 -1
  20. vectordb_bench/backend/dataset.py +100 -162
  21. vectordb_bench/backend/result_collector.py +2 -2
  22. vectordb_bench/backend/runner/mp_runner.py +29 -13
  23. vectordb_bench/backend/runner/serial_runner.py +98 -36
  24. vectordb_bench/backend/task_runner.py +43 -48
  25. vectordb_bench/frontend/components/check_results/charts.py +10 -21
  26. vectordb_bench/frontend/components/check_results/data.py +31 -15
  27. vectordb_bench/frontend/components/check_results/expanderStyle.py +37 -0
  28. vectordb_bench/frontend/components/check_results/filters.py +61 -33
  29. vectordb_bench/frontend/components/check_results/footer.py +8 -0
  30. vectordb_bench/frontend/components/check_results/headerIcon.py +8 -4
  31. vectordb_bench/frontend/components/check_results/nav.py +7 -6
  32. vectordb_bench/frontend/components/check_results/priceTable.py +3 -2
  33. vectordb_bench/frontend/components/check_results/stPageConfig.py +18 -0
  34. vectordb_bench/frontend/components/get_results/saveAsImage.py +50 -0
  35. vectordb_bench/frontend/components/run_test/autoRefresh.py +1 -1
  36. vectordb_bench/frontend/components/run_test/caseSelector.py +19 -16
  37. vectordb_bench/frontend/components/run_test/dbConfigSetting.py +20 -7
  38. vectordb_bench/frontend/components/run_test/dbSelector.py +5 -5
  39. vectordb_bench/frontend/components/run_test/hideSidebar.py +4 -6
  40. vectordb_bench/frontend/components/run_test/submitTask.py +16 -10
  41. vectordb_bench/frontend/const/dbCaseConfigs.py +291 -0
  42. vectordb_bench/frontend/const/dbPrices.py +6 -0
  43. vectordb_bench/frontend/const/styles.py +58 -0
  44. vectordb_bench/frontend/pages/{qps_with_price.py → quries_per_dollar.py} +24 -17
  45. vectordb_bench/frontend/pages/run_test.py +17 -11
  46. vectordb_bench/frontend/vdb_benchmark.py +19 -12
  47. vectordb_bench/metric.py +19 -10
  48. vectordb_bench/models.py +14 -40
  49. vectordb_bench/results/dbPrices.json +32 -0
  50. vectordb_bench/results/getLeaderboardData.py +52 -0
  51. vectordb_bench/results/leaderboard.json +1 -0
  52. vectordb_bench/results/{result_20230609_standard.json → result_20230705_standard.json} +1910 -897
  53. {vectordb_bench-0.0.1.dist-info → vectordb_bench-0.0.3.dist-info}/METADATA +107 -27
  54. vectordb_bench-0.0.3.dist-info/RECORD +67 -0
  55. vectordb_bench/frontend/const.py +0 -391
  56. vectordb_bench-0.0.1.dist-info/RECORD +0 -56
  57. {vectordb_bench-0.0.1.dist-info → vectordb_bench-0.0.3.dist-info}/LICENSE +0 -0
  58. {vectordb_bench-0.0.1.dist-info → vectordb_bench-0.0.3.dist-info}/WHEEL +0 -0
  59. {vectordb_bench-0.0.1.dist-info → vectordb_bench-0.0.3.dist-info}/entry_points.txt +0 -0
  60. {vectordb_bench-0.0.1.dist-info → vectordb_bench-0.0.3.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,6 @@
1
- from vectordb_bench.frontend.const import *
1
+ from vectordb_bench.frontend.const.styles import *
2
+ from vectordb_bench.backend.cases import CaseType
3
+ from vectordb_bench.frontend.const.dbCaseConfigs import *
2
4
 
3
5
 
4
6
  def caseSelector(st, activedDbList):
@@ -12,34 +14,35 @@ def caseSelector(st, activedDbList):
12
14
  unsafe_allow_html=True,
13
15
  )
14
16
 
15
- caseIsActived = {case["name"]: False for case in CASE_LIST}
16
- allCaseConfigs = {db: {case["name"]: {} for case in CASE_LIST} for db in DB_LIST}
17
- for case in CASE_LIST:
18
- caseItemContainer = st.container()
19
- caseIsActived[case["name"]] = caseItem(
20
- caseItemContainer, allCaseConfigs, case, activedDbList
21
- )
22
- if case.get("divider"):
17
+ caseIsActived = {case: False for case in CASE_LIST}
18
+ allCaseConfigs = {db: {case: {} for case in CASE_LIST} for db in DB_LIST}
19
+ for caseOrDivider in CASE_LIST_WITH_DIVIDER:
20
+ if caseOrDivider == DIVIDER:
23
21
  caseItemContainer.markdown(
24
22
  "<div style='border: 1px solid #cccccc60; margin-bottom: 24px;'></div>",
25
23
  unsafe_allow_html=True,
26
24
  )
27
- activedCaseList = [
28
- case["name"] for case in CASE_LIST if caseIsActived[case["name"]]
29
- ]
25
+ else:
26
+ case = caseOrDivider
27
+ caseItemContainer = st.container()
28
+ caseIsActived[case] = caseItem(
29
+ caseItemContainer, allCaseConfigs, case, activedDbList
30
+ )
31
+ activedCaseList = [case for case in CASE_LIST if caseIsActived[case]]
30
32
  return activedCaseList, allCaseConfigs
31
33
 
32
- def caseItem(st, allCaseConfigs, case, activedDbList):
33
- selected = st.checkbox(case["name"].value)
34
+
35
+ def caseItem(st, allCaseConfigs, case: CaseType, activedDbList):
36
+ selected = st.checkbox(case.case_name)
34
37
  st.markdown(
35
- f"<div style='color: #1D2939; margin: -8px 0 20px {CHECKBOX_INDENT}px; font-size: 14px;'>{case['intro']}</div>",
38
+ f"<div style='color: #1D2939; margin: -8px 0 20px {CHECKBOX_INDENT}px; font-size: 14px;'>{case.case_description}</div>",
36
39
  unsafe_allow_html=True,
37
40
  )
38
41
 
39
42
  if selected:
40
43
  caseConfigSettingContainer = st.container()
41
44
  caseConfigSetting(
42
- caseConfigSettingContainer, allCaseConfigs, case["name"], activedDbList
45
+ caseConfigSettingContainer, allCaseConfigs, case, activedDbList
43
46
  )
44
47
 
45
48
  return selected
@@ -1,4 +1,5 @@
1
- from vectordb_bench.frontend.const import *
1
+ from pydantic import ValidationError
2
+ from vectordb_bench.frontend.const.styles import *
2
3
  from vectordb_bench.frontend.utils import inputIsPassword
3
4
 
4
5
 
@@ -10,12 +11,25 @@ def dbConfigSettings(st, activedDbList):
10
11
  expander = st.expander("Configurations for the selected databases", True)
11
12
 
12
13
  dbConfigs = {}
14
+ isAllValid = True
13
15
  for activeDb in activedDbList:
14
16
  dbConfigSettingItemContainer = expander.container()
15
17
  dbConfig = dbConfigSettingItem(dbConfigSettingItemContainer, activeDb)
16
- dbConfigs[activeDb] = dbConfig
17
-
18
- return dbConfigs
18
+ try:
19
+ dbConfigs[activeDb] = activeDb.init_cls.config_cls()(**dbConfig)
20
+ except ValidationError as e:
21
+ isAllValid = False
22
+ errTexts = []
23
+ for err in e.raw_errors:
24
+ errLocs = err.loc_tuple()
25
+ errInfo = err.exc
26
+ errText = f"{', '.join(errLocs)} - {errInfo}"
27
+ errTexts.append(errText)
28
+
29
+ dbConfigSettingItemContainer.error(f"{'; '.join(errTexts)}")
30
+
31
+ return dbConfigs, isAllValid
32
+
19
33
 
20
34
  def dbConfigSettingItem(st, activeDb):
21
35
  st.markdown(
@@ -39,9 +53,8 @@ def dbConfigSettingItem(st, activeDb):
39
53
  value=value.get("default", ""),
40
54
  type="password" if inputIsPassword(key) else "default",
41
55
  )
42
- return dbConfigClass(**dbConfig)
56
+ return dbConfig
43
57
 
44
58
 
45
59
  def moveDBLabelToLast(propertiesItems):
46
- propertiesItems.sort(key=lambda x: 1 if x[0] == 'db_label' else 0)
47
-
60
+ propertiesItems.sort(key=lambda x: 1 if x[0] == "db_label" else 0)
@@ -1,5 +1,5 @@
1
-
2
- from vectordb_bench.frontend.const import *
1
+ from vectordb_bench.frontend.const.styles import *
2
+ from vectordb_bench.frontend.const.dbCaseConfigs import DB_LIST
3
3
 
4
4
 
5
5
  def dbSelector(st):
@@ -12,10 +12,10 @@ def dbSelector(st):
12
12
  "<div style='color: #647489; margin-bottom: 24px; margin-top: -12px;'>Choose at least one case you want to run the test for. </div>",
13
13
  unsafe_allow_html=True,
14
14
  )
15
-
15
+
16
16
  dbContainerColumns = st.columns(DB_SELECTOR_COLUMNS, gap="small")
17
17
  dbIsActived = {db: False for db in DB_LIST}
18
-
18
+
19
19
  # style - image; column gap; checkbox font;
20
20
  st.markdown(
21
21
  """
@@ -32,5 +32,5 @@ def dbSelector(st):
32
32
  dbIsActived[db] = column.checkbox(db.name)
33
33
  column.image(DB_TO_ICON.get(db, ""))
34
34
  activedDbList = [db for db in DB_LIST if dbIsActived[db]]
35
-
35
+
36
36
  return activedDbList
@@ -1,10 +1,8 @@
1
1
  def hideSidebar(st):
2
2
  st.markdown(
3
- "<style> div[data-testid='collapsedControl'] {display: none;} </style>",
4
- unsafe_allow_html=True,
5
- )
6
-
7
- st.markdown(
8
- "<style> .block-container { max-width: 1000px; } </style>",
3
+ """<style>
4
+ div[data-testid='collapsedControl'] {display: none;}
5
+ .block-container { max-width: 1000px; }
6
+ </style>""",
9
7
  unsafe_allow_html=True,
10
8
  )
@@ -1,9 +1,9 @@
1
1
  from datetime import datetime
2
- from vectordb_bench.frontend.const import *
2
+ from vectordb_bench.frontend.const.styles import *
3
3
  from vectordb_bench.interface import benchMarkRunner
4
4
 
5
5
 
6
- def submitTask(st, tasks):
6
+ def submitTask(st, tasks, isAllValid):
7
7
  st.markdown(
8
8
  "<div style='height: 24px;'></div>",
9
9
  unsafe_allow_html=True,
@@ -15,34 +15,36 @@ def submitTask(st, tasks):
15
15
  )
16
16
 
17
17
  taskLabel = taskLabelInput(st)
18
-
18
+
19
19
  st.markdown(
20
20
  "<div style='height: 24px;'></div>",
21
21
  unsafe_allow_html=True,
22
22
  )
23
23
 
24
24
  controlPanelContainer = st.container()
25
- controlPanel(controlPanelContainer, tasks, taskLabel)
25
+ controlPanel(controlPanelContainer, tasks, taskLabel, isAllValid)
26
26
 
27
27
 
28
28
  def taskLabelInput(st):
29
29
  defaultTaskLabel = datetime.now().strftime("%Y%m%d%H")
30
30
  columns = st.columns(TASK_LABEL_INPUT_COLUMNS)
31
- taskLabel = columns[0].text_input("task_label", defaultTaskLabel, label_visibility="collapsed")
31
+ taskLabel = columns[0].text_input(
32
+ "task_label", defaultTaskLabel, label_visibility="collapsed"
33
+ )
32
34
  return taskLabel
33
35
 
34
36
 
35
- def controlPanel(st, tasks, taskLabel):
37
+ def controlPanel(st, tasks, taskLabel, isAllValid):
36
38
  isRunning = benchMarkRunner.has_running()
37
39
  runHandler = lambda: benchMarkRunner.run(tasks, taskLabel)
38
40
  stopHandler = lambda: benchMarkRunner.stop_running()
39
-
41
+
40
42
  if isRunning:
41
43
  currentTaskId = benchMarkRunner.get_current_task_id()
42
44
  tasksCount = benchMarkRunner.get_tasks_count()
43
45
  text = f":running: Running Task {currentTaskId} / {tasksCount}"
44
46
  st.progress(currentTaskId / tasksCount, text=text)
45
-
47
+
46
48
  columns = st.columns(6)
47
49
  columns[0].button(
48
50
  "Run Your Test",
@@ -55,12 +57,16 @@ def controlPanel(st, tasks, taskLabel):
55
57
  on_click=stopHandler,
56
58
  type="primary",
57
59
  )
58
-
60
+
59
61
  else:
60
62
  errorText = benchMarkRunner.latest_error or ""
61
63
  if len(errorText) > 0:
62
64
  st.error(errorText)
63
- disabled = True if len(tasks) == 0 else False
65
+ disabled = True if len(tasks) == 0 or not isAllValid else False
66
+ if not isAllValid:
67
+ st.error("Make sure all config is valid.")
68
+ elif len(tasks) == 0:
69
+ st.warning("No tests to run.")
64
70
  st.button(
65
71
  "Run Your Test",
66
72
  disabled=disabled,
@@ -0,0 +1,291 @@
1
+ from enum import IntEnum
2
+ import typing
3
+ from pydantic import BaseModel
4
+ from vectordb_bench.backend.cases import CaseType
5
+ from vectordb_bench.backend.clients import DB
6
+ from vectordb_bench.backend.clients.api import IndexType
7
+
8
+ from vectordb_bench.models import CaseConfigParamType
9
+
10
+ MAX_STREAMLIT_INT = (1 << 53) - 1
11
+
12
+ DB_LIST = [d for d in DB]
13
+
14
+ DIVIDER = "DIVIDER"
15
+ CASE_LIST_WITH_DIVIDER = [
16
+ CaseType.Performance100M,
17
+ CaseType.Performance10M,
18
+ CaseType.Performance1M,
19
+ DIVIDER,
20
+ CaseType.Performance10M1P,
21
+ CaseType.Performance1M1P,
22
+ DIVIDER,
23
+ CaseType.Performance10M99P,
24
+ CaseType.Performance1M99P,
25
+ DIVIDER,
26
+ CaseType.CapacityDim960,
27
+ CaseType.CapacityDim128,
28
+ ]
29
+
30
+ CASE_LIST = [item for item in CASE_LIST_WITH_DIVIDER if isinstance(item, CaseType)]
31
+
32
+
33
+ class InputType(IntEnum):
34
+ Text = 20001
35
+ Number = 20002
36
+ Option = 20003
37
+
38
+
39
+ class CaseConfigInput(BaseModel):
40
+ label: CaseConfigParamType
41
+ inputType: InputType = InputType.Text
42
+ inputConfig: dict = {}
43
+ # todo type should be a function
44
+ isDisplayed: typing.Any = lambda x: True
45
+
46
+
47
+ CaseConfigParamInput_IndexType = CaseConfigInput(
48
+ label=CaseConfigParamType.IndexType,
49
+ inputType=InputType.Option,
50
+ inputConfig={
51
+ "options": [
52
+ IndexType.HNSW.value,
53
+ IndexType.IVFFlat.value,
54
+ IndexType.DISKANN.value,
55
+ IndexType.Flat.value,
56
+ IndexType.AUTOINDEX.value,
57
+ ],
58
+ },
59
+ )
60
+
61
+ CaseConfigParamInput_M = CaseConfigInput(
62
+ label=CaseConfigParamType.M,
63
+ inputType=InputType.Number,
64
+ inputConfig={
65
+ "min": 4,
66
+ "max": 64,
67
+ "value": 30,
68
+ },
69
+ isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
70
+ == IndexType.HNSW.value,
71
+ )
72
+
73
+ CaseConfigParamInput_EFConstruction_Milvus = CaseConfigInput(
74
+ label=CaseConfigParamType.EFConstruction,
75
+ inputType=InputType.Number,
76
+ inputConfig={
77
+ "min": 8,
78
+ "max": 512,
79
+ "value": 360,
80
+ },
81
+ isDisplayed=lambda config: config[CaseConfigParamType.IndexType]
82
+ == IndexType.HNSW.value,
83
+ )
84
+
85
+ CaseConfigParamInput_EFConstruction_Weaviate = CaseConfigInput(
86
+ label=CaseConfigParamType.EFConstruction,
87
+ inputType=InputType.Number,
88
+ inputConfig={
89
+ "min": 8,
90
+ "max": 512,
91
+ "value": 128,
92
+ },
93
+ )
94
+
95
+ CaseConfigParamInput_EFConstruction_ES = CaseConfigInput(
96
+ label=CaseConfigParamType.EFConstruction,
97
+ inputType=InputType.Number,
98
+ inputConfig={
99
+ "min": 8,
100
+ "max": 512,
101
+ "value": 360,
102
+ },
103
+ )
104
+
105
+ CaseConfigParamInput_M_ES = CaseConfigInput(
106
+ label=CaseConfigParamType.M,
107
+ inputType=InputType.Number,
108
+ inputConfig={
109
+ "min": 4,
110
+ "max": 64,
111
+ "value": 30,
112
+ },
113
+ )
114
+
115
+ CaseConfigParamInput_NumCandidates_ES = CaseConfigInput(
116
+ label=CaseConfigParamType.numCandidates,
117
+ inputType=InputType.Number,
118
+ inputConfig={
119
+ "min": 1,
120
+ "max": 10000,
121
+ "value": 100,
122
+ },
123
+ )
124
+
125
+ CaseConfigParamInput_EF_Milvus = CaseConfigInput(
126
+ label=CaseConfigParamType.EF,
127
+ inputType=InputType.Number,
128
+ inputConfig={
129
+ "min": 100,
130
+ "max": MAX_STREAMLIT_INT,
131
+ "value": 100,
132
+ },
133
+ isDisplayed=lambda config: config[CaseConfigParamType.IndexType]
134
+ == IndexType.HNSW.value,
135
+ )
136
+
137
+ CaseConfigParamInput_EF_Weaviate = CaseConfigInput(
138
+ label=CaseConfigParamType.EF,
139
+ inputType=InputType.Number,
140
+ inputConfig={
141
+ "min": -1,
142
+ "max": MAX_STREAMLIT_INT,
143
+ "value": -1,
144
+ },
145
+ )
146
+
147
+ CaseConfigParamInput_MaxConnections = CaseConfigInput(
148
+ label=CaseConfigParamType.MaxConnections,
149
+ inputType=InputType.Number,
150
+ inputConfig={"min": 1, "max": MAX_STREAMLIT_INT, "value": 64},
151
+ )
152
+
153
+ CaseConfigParamInput_SearchList = CaseConfigInput(
154
+ label=CaseConfigParamType.SearchList,
155
+ inputType=InputType.Number,
156
+ inputConfig={
157
+ "min": 100,
158
+ "max": MAX_STREAMLIT_INT,
159
+ "value": 100,
160
+ },
161
+ isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
162
+ == IndexType.DISKANN.value,
163
+ )
164
+
165
+ CaseConfigParamInput_Nlist = CaseConfigInput(
166
+ label=CaseConfigParamType.Nlist,
167
+ inputType=InputType.Number,
168
+ inputConfig={
169
+ "min": 1,
170
+ "max": 65536,
171
+ "value": 1000,
172
+ },
173
+ isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
174
+ == IndexType.IVFFlat.value,
175
+ )
176
+
177
+ CaseConfigParamInput_Nprobe = CaseConfigInput(
178
+ label=CaseConfigParamType.Nprobe,
179
+ inputType=InputType.Number,
180
+ inputConfig={
181
+ "min": 1,
182
+ "max": 65536,
183
+ "value": 10,
184
+ },
185
+ isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
186
+ == IndexType.IVFFlat.value,
187
+ )
188
+
189
+ CaseConfigParamInput_Lists = CaseConfigInput(
190
+ label=CaseConfigParamType.lists,
191
+ inputType=InputType.Number,
192
+ inputConfig={
193
+ "min": 1,
194
+ "max": 65536,
195
+ "value": 10,
196
+ },
197
+ )
198
+
199
+ CaseConfigParamInput_Probes = CaseConfigInput(
200
+ label=CaseConfigParamType.probes,
201
+ inputType=InputType.Number,
202
+ inputConfig={
203
+ "min": 1,
204
+ "max": 65536,
205
+ "value": 1,
206
+ },
207
+ )
208
+
209
+
210
+ MilvusLoadConfig = [
211
+ CaseConfigParamInput_IndexType,
212
+ CaseConfigParamInput_M,
213
+ CaseConfigParamInput_EFConstruction_Milvus,
214
+ CaseConfigParamInput_Nlist,
215
+ ]
216
+ MilvusPerformanceConfig = [
217
+ CaseConfigParamInput_IndexType,
218
+ CaseConfigParamInput_M,
219
+ CaseConfigParamInput_EFConstruction_Milvus,
220
+ CaseConfigParamInput_EF_Milvus,
221
+ CaseConfigParamInput_SearchList,
222
+ CaseConfigParamInput_Nlist,
223
+ CaseConfigParamInput_Nprobe,
224
+ ]
225
+
226
+ WeaviateLoadConfig = [
227
+ CaseConfigParamInput_MaxConnections,
228
+ CaseConfigParamInput_EFConstruction_Weaviate,
229
+ ]
230
+ WeaviatePerformanceConfig = [
231
+ CaseConfigParamInput_MaxConnections,
232
+ CaseConfigParamInput_EFConstruction_Weaviate,
233
+ CaseConfigParamInput_EF_Weaviate,
234
+ ]
235
+
236
+ ESLoadingConfig = [CaseConfigParamInput_EFConstruction_ES, CaseConfigParamInput_M_ES]
237
+ ESPerformanceConfig = [
238
+ CaseConfigParamInput_EFConstruction_ES,
239
+ CaseConfigParamInput_M_ES,
240
+ CaseConfigParamInput_NumCandidates_ES,
241
+ ]
242
+
243
+ PgVectorLoadingConfig = [CaseConfigParamInput_Lists]
244
+ PgVectorPerformanceConfig = [CaseConfigParamInput_Lists, CaseConfigParamInput_Probes]
245
+
246
+ CASE_CONFIG_MAP = {
247
+ DB.Milvus: {
248
+ CaseType.CapacityDim960: MilvusLoadConfig,
249
+ CaseType.CapacityDim128: MilvusLoadConfig,
250
+ CaseType.Performance100M: MilvusPerformanceConfig,
251
+ CaseType.Performance10M: MilvusPerformanceConfig,
252
+ CaseType.Performance1M: MilvusPerformanceConfig,
253
+ CaseType.Performance10M1P: MilvusPerformanceConfig,
254
+ CaseType.Performance1M1P: MilvusPerformanceConfig,
255
+ CaseType.Performance10M99P: MilvusPerformanceConfig,
256
+ CaseType.Performance1M99P: MilvusPerformanceConfig,
257
+ },
258
+ DB.WeaviateCloud: {
259
+ CaseType.CapacityDim960: WeaviateLoadConfig,
260
+ CaseType.CapacityDim128: WeaviateLoadConfig,
261
+ CaseType.Performance100M: WeaviatePerformanceConfig,
262
+ CaseType.Performance10M: WeaviatePerformanceConfig,
263
+ CaseType.Performance1M: WeaviatePerformanceConfig,
264
+ CaseType.Performance10M1P: WeaviatePerformanceConfig,
265
+ CaseType.Performance1M1P: WeaviatePerformanceConfig,
266
+ CaseType.Performance10M99P: WeaviatePerformanceConfig,
267
+ CaseType.Performance1M99P: WeaviatePerformanceConfig,
268
+ },
269
+ DB.ElasticCloud: {
270
+ CaseType.CapacityDim960: ESLoadingConfig,
271
+ CaseType.CapacityDim128: ESLoadingConfig,
272
+ CaseType.Performance100M: ESPerformanceConfig,
273
+ CaseType.Performance10M: ESPerformanceConfig,
274
+ CaseType.Performance1M: ESPerformanceConfig,
275
+ CaseType.Performance10M1P: ESPerformanceConfig,
276
+ CaseType.Performance1M1P: ESPerformanceConfig,
277
+ CaseType.Performance10M99P: ESPerformanceConfig,
278
+ CaseType.Performance1M99P: ESPerformanceConfig,
279
+ },
280
+ DB.PgVector: {
281
+ CaseType.CapacityDim960: PgVectorLoadingConfig,
282
+ CaseType.CapacityDim128: PgVectorLoadingConfig,
283
+ CaseType.Performance100M: PgVectorPerformanceConfig,
284
+ CaseType.Performance10M: PgVectorPerformanceConfig,
285
+ CaseType.Performance1M: PgVectorPerformanceConfig,
286
+ CaseType.Performance10M1P: PgVectorPerformanceConfig,
287
+ CaseType.Performance1M1P: PgVectorPerformanceConfig,
288
+ CaseType.Performance10M99P: PgVectorPerformanceConfig,
289
+ CaseType.Performance1M99P: PgVectorPerformanceConfig,
290
+ },
291
+ }
@@ -0,0 +1,6 @@
1
+ from vectordb_bench import config
2
+ import ujson
3
+ import pathlib
4
+
5
+ with open(pathlib.Path(config.RESULTS_LOCAL_DIR, "dbPrices.json")) as f:
6
+ DB_DBLABEL_TO_PRICE = ujson.load(f)
@@ -0,0 +1,58 @@
1
+ from vectordb_bench.models import DB
2
+
3
+ # style const
4
+ DB_SELECTOR_COLUMNS = 6
5
+ DB_CONFIG_SETTING_COLUMNS = 3
6
+ CASE_CONFIG_SETTING_COLUMNS = 4
7
+ CHECKBOX_INDENT = 30
8
+ TASK_LABEL_INPUT_COLUMNS = 2
9
+ CHECKBOX_MAX_COLUMNS = 4
10
+ DB_CONFIG_INPUT_MAX_COLUMNS = 2
11
+ CASE_CONFIG_INPUT_MAX_COLUMNS = 3
12
+ DB_CONFIG_INPUT_WIDTH_RADIO = 2
13
+ CASE_CONFIG_INPUT_WIDTH_RADIO = 0.98
14
+ CASE_INTRO_RATIO = 3
15
+ SIDEBAR_CONTROL_COLUMNS = 3
16
+ LEGEND_RECT_WIDTH = 24
17
+ LEGEND_RECT_HEIGHT = 16
18
+ LEGEND_TEXT_FONT_SIZE = 14
19
+
20
+ PATTERN_SHAPES = ["", "+", "\\", "x", ".", "|", "/", "-"]
21
+
22
+
23
+ def getPatternShape(i):
24
+ return PATTERN_SHAPES[i % len(PATTERN_SHAPES)]
25
+
26
+
27
+ # run_test page auto-refresh config
28
+ MAX_AUTO_REFRESH_COUNT = 999999
29
+ MAX_AUTO_REFRESH_INTERVAL = 5000 # 5s
30
+
31
+ PAGE_TITLE = "VectorDB Benchmark"
32
+ FAVICON = "https://assets.zilliz.com/favicon_f7f922fe27.png"
33
+ HEADER_ICON = "https://assets.zilliz.com/vdb_benchmark_db790b5387.png"
34
+
35
+ # RedisCloud icon: https://assets.zilliz.com/Redis_Cloud_74b8bfef39.png
36
+ # Elasticsearch icon: https://assets.zilliz.com/elasticsearch_beffeadc29.png
37
+ # Chroma icon: https://assets.zilliz.com/chroma_ceb3f06ed7.png
38
+ DB_TO_ICON = {
39
+ DB.Milvus: "https://assets.zilliz.com/milvus_c30b0d1994.png",
40
+ DB.ZillizCloud: "https://assets.zilliz.com/zilliz_5f4cc9b050.png",
41
+ DB.ElasticCloud: "https://assets.zilliz.com/Elatic_Cloud_dad8d6a3a3.png",
42
+ DB.Pinecone: "https://assets.zilliz.com/pinecone_94d8154979.png",
43
+ DB.QdrantCloud: "https://assets.zilliz.com/qdrant_b691674fcd.png",
44
+ DB.WeaviateCloud: "https://assets.zilliz.com/weaviate_4f6f171ebe.png",
45
+ DB.PgVector: "https://assets.zilliz.com/PG_Vector_d464f2ef5f.png",
46
+ }
47
+
48
+ # RedisCloud color: #0D6EFD
49
+ # Chroma color: #FFC107
50
+ COLOR_MAP = {
51
+ DB.Milvus.value: "#0DCAF0",
52
+ DB.ZillizCloud.value: "#0D6EFD",
53
+ DB.ElasticCloud.value: "#04D6C8",
54
+ DB.Pinecone.value: "#6610F2",
55
+ DB.QdrantCloud.value: "#D91AD9",
56
+ DB.WeaviateCloud.value: "#20C997",
57
+ DB.PgVector.value: "#4C779A",
58
+ }
@@ -1,32 +1,32 @@
1
1
  import streamlit as st
2
+ from vectordb_bench.frontend.components.check_results.footer import footer
3
+ from vectordb_bench.frontend.components.check_results.expanderStyle import initMainExpanderStyle
2
4
  from vectordb_bench.frontend.components.check_results.priceTable import priceTable
3
- from vectordb_bench.frontend.const import *
5
+ from vectordb_bench.frontend.components.check_results.stPageConfig import initResultsPageConfig
4
6
  from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
5
7
  from vectordb_bench.frontend.components.check_results.nav import NavToResults, NavToRunTest
6
8
  from vectordb_bench.frontend.components.check_results.charts import drawMetricChart
7
9
  from vectordb_bench.frontend.components.check_results.filters import getshownData
10
+ from vectordb_bench.frontend.components.get_results.saveAsImage import getResults
11
+ from vectordb_bench.frontend.const.styles import *
8
12
  from vectordb_bench.interface import benchMarkRunner
13
+ from vectordb_bench.metric import QURIES_PER_DOLLAR_METRIC
9
14
 
10
15
 
11
16
  def main():
12
- st.set_page_config(
13
- page_title="VectorDB Benchmark",
14
- page_icon="https://assets.zilliz.com/favicon_f7f922fe27.png",
15
- # layout="wide",
16
- # initial_sidebar_state="collapsed",
17
- )
18
-
17
+ # set page config
18
+ initResultsPageConfig(st)
19
+
19
20
  # header
20
21
  drawHeaderIcon(st)
21
22
 
22
23
  allResults = benchMarkRunner.get_results()
23
24
 
24
- st.title("Vector DB Benchmark (QPS / Price)")
25
- st.subheader("Price List")
25
+ st.title("Vector DB Benchmark (QP$)")
26
26
 
27
27
  # results selector
28
28
  resultSelectorContainer = st.sidebar.container()
29
- shownData, failedTasks, showCases = getshownData(allResults, resultSelectorContainer)
29
+ shownData, _, showCases = getshownData(allResults, resultSelectorContainer)
30
30
 
31
31
  resultSelectorContainer.divider()
32
32
 
@@ -34,27 +34,34 @@ def main():
34
34
  navContainer = st.sidebar.container()
35
35
  NavToRunTest(navContainer)
36
36
  NavToResults(navContainer)
37
-
37
+
38
+ # save or share
39
+ resultesContainer = st.sidebar.container()
40
+ getResults(resultesContainer, "vectordb_bench_qp$")
41
+
38
42
  # price table
43
+ initMainExpanderStyle(st)
39
44
  priceTableContainer = st.container()
40
45
  priceMap = priceTable(priceTableContainer, shownData)
41
46
 
42
47
  # charts
43
48
  for case in showCases:
44
- chartContainer = st.container()
45
- data = [data for data in shownData if data["case"] == case]
49
+ data = [data for data in shownData if data["case_name"] == case.name]
46
50
  dataWithMetric = []
47
- metric = "qps_per_dollar (qps / price)"
51
+ metric = QURIES_PER_DOLLAR_METRIC
48
52
  for d in data:
49
53
  qps = d.get("qps", 0)
50
54
  price = priceMap.get(d["db"], {}).get(d["db_label"], 0)
51
55
  if qps > 0 and price > 0:
52
- d[metric] = d["qps"] / price
56
+ d[metric] = d["qps"] / price * 3.6
53
57
  dataWithMetric.append(d)
54
58
  if len(dataWithMetric) > 0:
55
- chartContainer.subheader(case)
59
+ chartContainer = st.expander(case.name, True)
56
60
  drawMetricChart(data, metric, chartContainer)
57
61
 
62
+ # footer
63
+ footer(st.container())
64
+
58
65
 
59
66
  if __name__ == "__main__":
60
67
  main()