vectordb-bench 0.0.30__py3-none-any.whl → 1.0.0__py3-none-any.whl

vectordb_bench/frontend/pages/concurrent.py

@@ -4,6 +4,7 @@ from vectordb_bench.frontend.components.check_results.headerIcon import drawHead
 from vectordb_bench.frontend.components.check_results.nav import (
     NavToResults,
     NavToRunTest,
+    NavToPages,
 )
 from vectordb_bench.frontend.components.check_results.filters import getshownData
 from vectordb_bench.frontend.components.concurrent.charts import drawChartsByCase
@@ -25,6 +26,9 @@ def main():
     # header
     drawHeaderIcon(st)
 
+    # navigate
+    NavToPages(st)
+
     allResults = benchmark_runner.get_results()
 
     def check_conc_data(res: TestResult):
@@ -42,7 +46,7 @@ def main():
 
     # results selector
     resultSelectorContainer = st.sidebar.container()
-    shownData, _, showCaseNames = getshownData(checkedResults, resultSelectorContainer)
+    shownData, _, showCaseNames = getshownData(resultSelectorContainer, checkedResults)
 
     resultSelectorContainer.divider()
 

vectordb_bench/frontend/pages/custom.py

@@ -1,6 +1,7 @@
 from functools import partial
 import streamlit as st
 from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
+from vectordb_bench.frontend.components.check_results.nav import NavToPages
 from vectordb_bench.frontend.components.custom.displayCustomCase import (
     displayCustomCase,
 )
@@ -49,6 +50,9 @@ def main():
     # init style
     initStyle(st)
 
+    # navigate
+    NavToPages(st)
+
     st.title("Custom Dataset")
     displayParams(st)
     customCaseManager = CustomCaseManager()

vectordb_bench/frontend/pages/label_filter.py

@@ -0,0 +1,56 @@
+import streamlit as st
+from vectordb_bench.backend.filter import FilterOp
+from vectordb_bench.frontend.components.check_results.footer import footer
+from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
+from vectordb_bench.frontend.components.check_results.nav import (
+    NavToQuriesPerDollar,
+    NavToRunTest,
+    NavToPages,
+)
+from vectordb_bench.frontend.components.label_filter.charts import drawCharts
+from vectordb_bench.frontend.components.check_results.filters import getshownData
+from vectordb_bench.frontend.config.styles import FAVICON
+from vectordb_bench.interface import benchmark_runner
+
+
+def main():
+    # set page config
+    st.set_page_config(
+        page_title="Label Filter",
+        page_icon=FAVICON,
+        layout="wide",
+        # initial_sidebar_state="collapsed",
+    )
+
+    # header
+    drawHeaderIcon(st)
+
+    # navigate
+    NavToPages(st)
+
+    allResults = benchmark_runner.get_results()
+
+    st.title("Vector Database Benchmark (Label Filter)")
+
+    # results selector and filter
+    resultSelectorContainer = st.sidebar.container()
+    shownData, failedTasks, showCaseNames = getshownData(
+        resultSelectorContainer, allResults, filter_type=FilterOp.StrEqual
+    )
+
+    resultSelectorContainer.divider()
+
+    # nav
+    navContainer = st.sidebar.container()
+    NavToRunTest(navContainer)
+    NavToQuriesPerDollar(navContainer)
+
+    # charts
+    drawCharts(st, shownData)
+
+    # footer
+    footer(st.container())
+
+
+if __name__ == "__main__":
+    main()

vectordb_bench/frontend/pages/quries_per_dollar.py

@@ -9,6 +9,7 @@ from vectordb_bench.frontend.components.check_results.stPageConfig import (
 )
 from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
 from vectordb_bench.frontend.components.check_results.nav import (
+    NavToPages,
     NavToResults,
     NavToRunTest,
 )
@@ -27,13 +28,16 @@ def main():
     # header
     drawHeaderIcon(st)
 
+    # navigate
+    NavToPages(st)
+
     allResults = benchmark_runner.get_results()
 
     st.title("Vector DB Benchmark (QP$)")
 
     # results selector
     resultSelectorContainer = st.sidebar.container()
-    shownData, _, showCaseNames = getshownData(allResults, resultSelectorContainer)
+    shownData, _, showCaseNames = getshownData(resultSelectorContainer, allResults)
 
     resultSelectorContainer.divider()
 

vectordb_bench/frontend/pages/results.py

@@ -0,0 +1,60 @@
+import streamlit as st
+from vectordb_bench.frontend.components.check_results.footer import footer
+from vectordb_bench.frontend.components.check_results.stPageConfig import (
+    initResultsPageConfig,
+)
+from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
+from vectordb_bench.frontend.components.check_results.nav import (
+    NavToQuriesPerDollar,
+    NavToRunTest,
+    NavToPages,
+)
+from vectordb_bench.frontend.components.check_results.charts import drawCharts
+from vectordb_bench.frontend.components.check_results.filters import getshownData
+from vectordb_bench.frontend.components.get_results.saveAsImage import getResults
+
+from vectordb_bench.interface import benchmark_runner
+
+
+def main():
+    # set page config
+    initResultsPageConfig(st)
+
+    # header
+    drawHeaderIcon(st)
+
+    # navigate
+    NavToPages(st)
+
+    allResults = benchmark_runner.get_results()
+
+    st.title("Vector Database Benchmark")
+    st.caption(
+        "Except for zillizcloud-v2024.1, which was tested in _January 2024_, all other tests were completed before _August 2023_."
+    )
+    st.caption("All tested milvus are in _standalone_ mode.")
+
+    # results selector and filter
+    resultSelectorContainer = st.sidebar.container()
+    shownData, failedTasks, showCaseNames = getshownData(resultSelectorContainer, allResults)
+
+    resultSelectorContainer.divider()
+
+    # nav
+    navContainer = st.sidebar.container()
+    NavToRunTest(navContainer)
+    NavToQuriesPerDollar(navContainer)
+
+    # save or share
+    resultesContainer = st.sidebar.container()
+    getResults(resultesContainer, "vectordb_bench")
+
+    # charts
+    drawCharts(st, shownData, failedTasks, showCaseNames)
+
+    # footer
+    footer(st.container())
+
+
+if __name__ == "__main__":
+    main()

vectordb_bench/frontend/pages/run_test.py

@@ -7,7 +7,7 @@ from vectordb_bench.frontend.components.run_test.generateTasks import generate_t
 from vectordb_bench.frontend.components.run_test.hideSidebar import hideSidebar
 from vectordb_bench.frontend.components.run_test.initStyle import initStyle
 from vectordb_bench.frontend.components.run_test.submitTask import submitTask
-from vectordb_bench.frontend.components.check_results.nav import NavToResults
+from vectordb_bench.frontend.components.check_results.nav import NavToResults, NavToPages
 from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
 from vectordb_bench.frontend.components.check_results.stPageConfig import initRunTestPageConfig
 
@@ -25,8 +25,8 @@ def main():
     # hide sidebar
     hideSidebar(st)
 
-    # nav to results
-    NavToResults(st)
+    # navigate
+    NavToPages(st)
 
     # header
     st.title("Run Your Test")

vectordb_bench/frontend/pages/streaming.py

@@ -0,0 +1,135 @@
+import logging
+import streamlit as st
+from vectordb_bench.backend.cases import CaseLabel
+from vectordb_bench.frontend.components.check_results.footer import footer
+from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
+from vectordb_bench.frontend.components.check_results.nav import (
+    NavToResults,
+    NavToRunTest,
+    NavToPages,
+)
+from vectordb_bench.frontend.components.check_results.filters import getshownData
+from vectordb_bench.frontend.components.streaming.charts import drawChartsByCase
+from vectordb_bench.frontend.components.get_results.saveAsImage import getResults
+from vectordb_bench.frontend.components.streaming.data import DisplayedMetric
+from vectordb_bench.frontend.config.styles import FAVICON
+from vectordb_bench.interface import benchmark_runner
+from vectordb_bench.models import CaseResult, TestResult
+
+log = logging.getLogger("vectordb_bench")
+
+
+def main():
+    # set page config
+    st.set_page_config(
+        page_title="VDBB Streaming Perf",
+        page_icon=FAVICON,
+        layout="wide",
+        # initial_sidebar_state="collapsed",
+    )
+
+    # header
+    drawHeaderIcon(st)
+
+    # navigate
+    NavToPages(st)
+
+    allResults = benchmark_runner.get_results()
+
+    def check_streaming_data(res: TestResult):
+        case_results = res.results
+        flag = False
+        for case_result in case_results:
+            if case_result.task_config.case_config.case.label == CaseLabel.Streaming:
+                flag = True
+
+        return flag
+
+    checkedResults = [res for res in allResults if check_streaming_data(res)]
+
+    st.title("VDBBench - Streaming Performance")
+
+    # results selector
+    resultSelectorContainer = st.sidebar.container()
+
+    def case_results_filter(case_result: CaseResult) -> bool:
+        return len(case_result.metrics.st_search_stage_list) > 0
+
+    shownData, _, showCaseNames = getshownData(
+        resultSelectorContainer, checkedResults, case_results_filter=case_results_filter
+    )
+
+    resultSelectorContainer.divider()
+
+    # nav
+    navContainer = st.sidebar.container()
+    NavToRunTest(navContainer)
+    NavToResults(navContainer)
+
+    # save or share
+    resultesContainer = st.sidebar.container()
+    getResults(resultesContainer, "vectordb_bench_streaming")
+
+    # # main
+    # latency_type = st.radio("Latency Type", options=["latency_p99", "latency_avg"])
+    st.markdown("Tests search performance with a **stable** and **fixed** insertion rate.")
+    control_panel = st.columns(3)
+    compared_with_optimized = control_panel[0].toggle(
+        "Compare with **optimezed** performance.",
+        value=True,
+        help="VectorDB is allowed to do **optimizations** after all insertions done and then test search performance.",
+    )
+    x_use_actual_time = control_panel[0].toggle(
+        "Use **actual time** as X-axis instead of search stage.",
+        value=False,
+        help="Since vdbbench inserts may be faster than vetordb can process them, the time it actually reaches search_stage may have different delays.",
+    )
+    accuracy_metric = DisplayedMetric.recall
+    show_ndcg = control_panel[1].toggle(
+        "Show **NDCG** instead of Recall.",
+        value=False,
+        help="A more appropriate indicator to measure ANN search accuracy than Recall.",
+    )
+    need_adjust = control_panel[1].toggle(
+        "Adjust the NDCG/Recall value based on the search stage.",
+        value=True,
+        help="NDCG/Recall is calculated using the ground truth file of the **entire** database, **divided by the search stage** to simulate the actual value.",
+    )
+    if show_ndcg:
+        if need_adjust:
+            accuracy_metric = DisplayedMetric.adjusted_ndcg
+        else:
+            accuracy_metric = DisplayedMetric.ndcg
+    else:
+        if need_adjust:
+            accuracy_metric = DisplayedMetric.adjusted_recall
+    line_chart_displayed_y_metrics: list[tuple[DisplayedMetric, str]] = [
+        (
+            DisplayedMetric.qps,
+            "max-qps of increasing **concurrency search** tests in each search stage.",
+        ),
+        (accuracy_metric, "calculated in each search_stage."),
+        (
+            DisplayedMetric.latency_p99,
+            "serial lantency (p99) of **serial search** tests in each search stage.",
+        ),
+    ]
+    line_chart_displayed_x_metric = DisplayedMetric.search_stage
+    if x_use_actual_time:
+        line_chart_displayed_x_metric = DisplayedMetric.search_time
+
+    drawChartsByCase(
+        st.container(),
+        shownData,
+        showCaseNames,
+        with_last_optimized_data=compared_with_optimized,
+        line_chart_displayed_x_metric=line_chart_displayed_x_metric,
+        line_chart_displayed_y_metrics=line_chart_displayed_y_metrics,
+    )
+
+    # footer
+    footer(st.container())
+
+
+if __name__ == "__main__":
+    main()

vectordb_bench/frontend/pages/tables.py

@@ -1,5 +1,6 @@
 import streamlit as st
 from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
+from vectordb_bench.frontend.components.check_results.nav import NavToPages
 from vectordb_bench.frontend.components.tables.data import getNewResults
 from vectordb_bench.frontend.config.styles import FAVICON
 
@@ -16,6 +17,9 @@ def main():
     # header
     drawHeaderIcon(st)
 
+    # navigate
+    NavToPages(st)
+
     df = getNewResults()
     st.dataframe(df, height=800)
 

vectordb_bench/frontend/vdb_benchmark.py

@@ -1,55 +1,30 @@
 import streamlit as st
-from vectordb_bench.frontend.components.check_results.footer import footer
-from vectordb_bench.frontend.components.check_results.stPageConfig import (
-    initResultsPageConfig,
-)
 from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
-from vectordb_bench.frontend.components.check_results.nav import (
-    NavToQuriesPerDollar,
-    NavToRunTest,
-)
-from vectordb_bench.frontend.components.check_results.charts import drawCharts
-from vectordb_bench.frontend.components.check_results.filters import getshownData
-from vectordb_bench.frontend.components.get_results.saveAsImage import getResults
-
-from vectordb_bench.interface import benchmark_runner
+from vectordb_bench.frontend.components.custom.initStyle import initStyle
+from vectordb_bench.frontend.components.welcome.explainPrams import explainPrams
+from vectordb_bench.frontend.components.welcome.welcomePrams import welcomePrams
+from vectordb_bench.frontend.config.styles import FAVICON, PAGE_TITLE
 
 
 def main():
-    # set page config
-    initResultsPageConfig(st)
+    st.set_page_config(
+        page_title=PAGE_TITLE,
+        page_icon=FAVICON,
+        layout="wide",
+        initial_sidebar_state="collapsed",
+    )
 
     # header
     drawHeaderIcon(st)
 
-    allResults = benchmark_runner.get_results()
-
-    st.title("Vector Database Benchmark")
-    st.caption(
-        "Except for zillizcloud-v2024.1, which was tested in _January 2024_, all other tests were completed before _August 2023_."
-    )
-    st.caption("All tested milvus are in _standalone_ mode.")
-
-    # results selector and filter
-    resultSelectorContainer = st.sidebar.container()
-    shownData, failedTasks, showCaseNames = getshownData(allResults, resultSelectorContainer)
-
-    resultSelectorContainer.divider()
-
-    # nav
-    navContainer = st.sidebar.container()
-    NavToRunTest(navContainer)
-    NavToQuriesPerDollar(navContainer)
-
-    # save or share
-    resultesContainer = st.sidebar.container()
-    getResults(resultesContainer, "vectordb_bench")
+    # init style
+    initStyle(st)
 
-    # charts
-    drawCharts(st, shownData, failedTasks, showCaseNames)
+    # page
+    welcomePrams(st)
 
-    # footer
-    footer(st.container())
+    # description
+    explainPrams(st)
 
 
 if __name__ == "__main__":

vectordb_bench/interface.py

@@ -12,7 +12,7 @@ from multiprocessing.connection import Connection
 import psutil
 
 from . import config
-from .backend.assembler import Assembler
+from .backend.assembler import Assembler, FilterNotSupportedError
 from .backend.data_source import DatasetSource
 from .backend.result_collector import ResultCollector
 from .backend.task_runner import TaskRunner
@@ -88,6 +88,10 @@ class BenchMarkRunner:
             log.warning(msg)
             self.latest_error = msg
             return True
+        except FilterNotSupportedError as e:
+            log.warning(e.args[0])
+            self.latest_error = e.args[0]
+            return True
 
         return self._run_async(send_conn)
 
@@ -97,7 +101,7 @@ class BenchMarkRunner:
         return ResultCollector.collect(target_dir)
 
     def _try_get_signal(self):
-        if self.receive_conn and self.receive_conn.poll():
+        while self.receive_conn and self.receive_conn.poll():
             sig, received = self.receive_conn.recv()
             log.debug(f"Sigal received to process: {sig}, {received}")
             if sig == SIGNAL.ERROR:

vectordb_bench/metric.py

@@ -13,8 +13,12 @@ class Metric:
     # for load cases
     max_load_count: int = 0
 
+    # for both performace and streaming cases
+    insert_duration: float = 0.0
+    optimize_duration: float = 0.0
+    load_duration: float = 0.0  # insert + optimize
+
     # for performance cases
-    load_duration: float = 0.0  # duration to load all dataset into DB
     qps: float = 0.0
     serial_latency_p99: float = 0.0
     recall: float = 0.0
@@ -24,6 +28,16 @@ class Metric:
     conc_latency_p99_list: list[float] = field(default_factory=list)
     conc_latency_avg_list: list[float] = field(default_factory=list)
 
+    # for streaming cases
+    st_ideal_insert_duration: int = 0
+    st_search_stage_list: list[int] = field(default_factory=list)
+    st_search_time_list: list[float] = field(default_factory=list)
+    st_max_qps_list_list: list[float] = field(default_factory=list)
+    st_recall_list: list[float] = field(default_factory=list)
+    st_ndcg_list: list[float] = field(default_factory=list)
+    st_serial_latency_p99_list: list[float] = field(default_factory=list)
+    st_conc_failed_rate_list: list[float] = field(default_factory=list)
+
 
 QURIES_PER_DOLLAR_METRIC = "QP$ (Quries per Dollar)"
 LOAD_DURATION_METRIC = "load_duration"

vectordb_bench/models.py

@@ -1,13 +1,13 @@
 import logging
 import pathlib
 from datetime import date, datetime
-from enum import Enum, StrEnum, auto
+from enum import Enum, StrEnum
 from typing import Self
 
 import ujson
 
 from . import config
-from .backend.cases import CaseType
+from .backend.cases import Case, CaseType
 from .backend.clients import (
     DB,
     DBCaseConfig,
@@ -112,10 +112,20 @@ class CaseConfigParamType(Enum):
     index_thread_qty = "index_thread_qty"
     engine_name = "engine_name"
     metric_type_name = "metric_type_name"
-
-    # mongodb params
     mongodb_quantization_type = "quantization"
     mongodb_num_candidates_ratio = "num_candidates_ratio"
+    use_partition_key = "use_partition_key"
+    refresh_interval = "refresh_interval"
+    use_rescore = "use_rescore"
+    oversample_ratio = "oversample_ratio"
+    use_routing = "use_routing"
+
+    dataset_with_size_type = "dataset_with_size_type"
+    insert_rate = "insert_rate"
+    search_stages = "search_stages"
+    concurrencies = "concurrencies"
+    optimize_after_write = "optimize_after_write"
+    read_dur_after_write = "read_dur_after_write"
 
 
 class CustomizedCase(BaseModel):
@@ -151,14 +161,22 @@ class CaseConfig(BaseModel):
     def __hash__(self) -> int:
         return hash(self.json())
 
+    @property
+    def case(self) -> Case:
+        return self.case_id.case_cls(self.custom_case)
+
+    @property
+    def case_name(self) -> str:
+        return self.case.name
+
 
 class TaskStage(StrEnum):
     """Enumerations of various stages of the task"""
 
-    DROP_OLD = auto()
-    LOAD = auto()
-    SEARCH_SERIAL = auto()
-    SEARCH_CONCURRENT = auto()
+    DROP_OLD = "drop_old"
+    LOAD = "load"
+    SEARCH_SERIAL = "search_serial"
+    SEARCH_CONCURRENT = "search_concurrent"
 
     def __repr__(self) -> str:
         return str.__repr__(self.value)
@@ -299,12 +317,14 @@ class TestResult(BaseModel):
             key=lambda x: (
                 x.task_config.db.name,
                 x.task_config.db_config.db_label,
-                x.task_config.case_config.case_id.name,
+                x.task_config.case_config.case_name,
             ),
             reverse=True,
         )
 
         filtered_results = [r for r in sorted_results if not filter_list or r.task_config.db not in filter_list]
+        if len(filtered_results) == 0:
+            return
 
         def append_return(x: any, y: any):
             x.append(y)
@@ -312,7 +332,7 @@ class TestResult(BaseModel):
 
         max_db = max(map(len, [f.task_config.db.name for f in filtered_results]))
         max_db_labels = max(map(len, [f.task_config.db_config.db_label for f in filtered_results])) + 3
-        max_case = max(map(len, [f.task_config.case_config.case_id.name for f in filtered_results]))
+        max_case = max(map(len, [f.task_config.case_config.case_name for f in filtered_results]))
         max_load_dur = max(map(len, [str(f.metrics.load_duration) for f in filtered_results])) + 3
         max_qps = max(map(len, [str(f.metrics.qps) for f in filtered_results])) + 3
         max_recall = max(map(len, [str(f.metrics.recall) for f in filtered_results])) + 3
@@ -366,7 +386,7 @@ class TestResult(BaseModel):
                 % (
                     f.task_config.db.name,
                     f.task_config.db_config.db_label,
-                    f.task_config.case_config.case_id.name,
+                    f.task_config.case_config.case_name,
                     self.task_label,
                     f.metrics.load_duration,
                     f.metrics.qps,