vectordb-bench 0.0.1__1-py3-none-any.whl

vectordb_bench/frontend/components/run_test/submitTask.py

@@ -0,0 +1,69 @@
+from datetime import datetime
+from vectordb_bench.frontend.const import *
+from vectordb_bench.interface import benchMarkRunner
+
+
+def submitTask(st, tasks):
+    st.markdown(
+        "<div style='height: 24px;'></div>",
+        unsafe_allow_html=True,
+    )
+    st.subheader("STEP 3: Task Label")
+    st.markdown(
+        "<div style='color: #647489; margin-bottom: 20px; margin-top: -12px;'>This description is used to mark the result. </div>",
+        unsafe_allow_html=True,
+    )
+
+    taskLabel = taskLabelInput(st)
+    
+    st.markdown(
+        "<div style='height: 24px;'></div>",
+        unsafe_allow_html=True,
+    )
+
+    controlPanelContainer = st.container()
+    controlPanel(controlPanelContainer, tasks, taskLabel)
+
+
+def taskLabelInput(st):
+    defaultTaskLabel = datetime.now().strftime("%Y%m%d%H")
+    columns = st.columns(TASK_LABEL_INPUT_COLUMNS)
+    taskLabel = columns[0].text_input("task_label", defaultTaskLabel, label_visibility="collapsed")
+    return taskLabel
+
+
+def controlPanel(st, tasks, taskLabel):
+    isRunning = benchMarkRunner.has_running()
+    runHandler = lambda: benchMarkRunner.run(tasks, taskLabel)
+    stopHandler = lambda: benchMarkRunner.stop_running()
+    
+    if isRunning:
+        currentTaskId = benchMarkRunner.get_current_task_id()
+        tasksCount = benchMarkRunner.get_tasks_count()
+        text = f":running: Running Task {currentTaskId} / {tasksCount}"
+        st.progress(currentTaskId / tasksCount, text=text)
+        
+        columns = st.columns(6)
+        columns[0].button(
+            "Run Your Test",
+            disabled=True,
+            on_click=runHandler,
+            type="primary",
+        )
+        columns[1].button(
+            "Stop",
+            on_click=stopHandler,
+            type="primary",
+        )
+        
+    else:
+        errorText = benchMarkRunner.latest_error or ""
+        if len(errorText) > 0:
+            st.error(errorText)
+        disabled = True if len(tasks) == 0 else False
+        st.button(
+            "Run Your Test",
+            disabled=disabled,
+            on_click=runHandler,
+            type="primary",
+        )

vectordb_bench/frontend/const.py

@@ -0,0 +1,391 @@
+from enum import IntEnum
+from vectordb_bench.models import DB, CaseType, IndexType, CaseConfigParamType
+from pydantic import BaseModel
+import typing
+
+# style const
+DB_SELECTOR_COLUMNS = 6
+DB_CONFIG_SETTING_COLUMNS = 3
+CASE_CONFIG_SETTING_COLUMNS = 4
+CHECKBOX_INDENT = 30
+TASK_LABEL_INPUT_COLUMNS = 2
+CHECKBOX_MAX_COLUMNS = 4
+DB_CONFIG_INPUT_MAX_COLUMNS = 2
+CASE_CONFIG_INPUT_MAX_COLUMNS = 3
+DB_CONFIG_INPUT_WIDTH_RADIO = 2
+CASE_CONFIG_INPUT_WIDTH_RADIO = 0.98
+CASE_INTRO_RATIO = 3
+MAX_STREAMLIT_INT = (1 << 53) - 1
+
+LEGEND_RECT_WIDTH = 24
+LEGEND_RECT_HEIGHT = 16
+LEGEND_TEXT_FONT_SIZE = 14
+
+PATTERN_SHAPES = ["", "+", "\\", "x", ".", "|", "/", "-"]
+
+
+def getPatternShape(i):
+    return PATTERN_SHAPES[i % len(PATTERN_SHAPES)]
+
+
+MAX_AUTO_REFRESH_COUNT = 999999
+MAX_AUTO_REFRESH_INTERVAL = 5000  # 2s
+
+
+DB_LIST = [d for d in DB]
+
+DB_TO_ICON = {
+    DB.Milvus: "https://assets.zilliz.com/milvus_c30b0d1994.png",
+    DB.ZillizCloud: "https://assets.zilliz.com/zilliz_5f4cc9b050.png",
+    DB.ElasticCloud: "https://assets.zilliz.com/elasticsearch_beffeadc29.png",
+    DB.Pinecone: "https://assets.zilliz.com/pinecone_94d8154979.png",
+    DB.QdrantCloud: "https://assets.zilliz.com/qdrant_b691674fcd.png",
+    DB.WeaviateCloud: "https://assets.zilliz.com/weaviate_4f6f171ebe.png",
+}
+
+COLOR_MAP = {
+    DB.Milvus.value: "#0DCAF0",
+    DB.ZillizCloud.value: "#0D6EFD",
+    DB.ElasticCloud.value: "#fdc613",
+    DB.Pinecone.value: "#6610F2",
+    DB.QdrantCloud.value: "#D91AD9",
+    DB.WeaviateCloud.value: "#20C997",
+}
+
+CASE_LIST = [
+    {
+        "name": CaseType.CapacitySDim,
+        "intro": """This case tests the vector database's loading capacity by repeatedly inserting small-dimension vectors (SIFT 100K vectors, <b>128 dimensions</b>) until it is fully loaded.  
+Number of inserted vectors will be reported.""",
+    },
+    {
+        "name": CaseType.CapacityLDim,
+        "divider": True,
+        "intro": """This case tests the vector database's loading capacity by repeatedly inserting large-dimension vectors (GIST 100K vectors, <b>960 dimensions</b>) until it is fully loaded.  
+Number of inserted vectors will be reported.
+""",
+    },
+    {
+        "name": CaseType.PerformanceSZero,
+        "intro": """This case tests the search performance of a vector database with a small dataset (<b>Cohere 100K vectors</b>, 768 dimensions) at varying parallel levels.
+Results will show index building time, recall, and maximum QPS.""",
+    },
+    {
+        "name": CaseType.PerformanceMZero,
+        "intro": """This case tests the search performance of a vector database with a medium dataset (<b>Cohere 1M vectors</b>, 768 dimensions) at varying parallel levels.
+Results will show index building time, recall, and maximum QPS.""",
+    },
+    {
+        "name": CaseType.PerformanceLZero,
+        "intro": """This case tests the search performance of a vector database with a large dataset (<b>Cohere 10M vectors</b>, 768 dimensions) at varying parallel levels.
+Results will show index building time, recall, and maximum QPS.""",
+    },
+    {
+        "name": CaseType.Performance100M,
+        "divider": True,
+        "intro": """This case tests the search performance of a vector database with a large 100M dataset (<b>LAION 100M vectors</b>, 768 dimensions), at varying parallel levels.
+Results will show index building time, recall, and maximum QPS.""",
+    },
+    {
+        "name": CaseType.PerformanceSLow,
+        "intro": """This case tests the search performance of a vector database with a small dataset (<b>Cohere 100K vectors</b>, 768 dimensions) under a low filtering rate (<b>1% vectors</b>), at varying parallel levels.
+Results will show index building time, recall, and maximum QPS.""",
+    },
+    {
+        "name": CaseType.PerformanceMLow,
+        "intro": """This case tests the search performance of a vector database with a medium dataset (<b>Cohere 1M vectors</b>, 768 dimensions) under a low filtering rate (<b>1% vectors</b>), at varying parallel levels.
+Results will show index building time, recall, and maximum QPS.""",
+    },
+    {
+        "name": CaseType.PerformanceLLow,
+        "intro": """This case tests the search performance of a vector database with a large dataset (<b>Cohere 10M vectors</b>, 768 dimensions) under a low filtering rate (<b>1% vectors</b>), at varying parallel levels.
+Results will show index building time, recall, and maximum QPS.""",
+    },
+    {
+        "name": CaseType.PerformanceSHigh,
+        "intro": """This case tests the search performance of a vector database with a small dataset (<b>Cohere 100K vectors</b>, 768 dimensions) under a high filtering rate (<b>99% vectors</b>), at varying parallel levels.
+Results will show index building time, recall, and maximum QPS.""",
+    },
+    {
+        "name": CaseType.PerformanceMHigh,
+        "intro": """This case tests the search performance of a vector database with a medium dataset (<b>Cohere 1M vectors</b>, 768 dimensions) under a high filtering rate (<b>99% vectors</b>), at varying parallel levels.
+Results will show index building time, recall, and maximum QPS.""",
+    },
+    {
+        "name": CaseType.PerformanceLHigh,
+        "intro": """This case tests the search performance of a vector database with a large dataset (<b>Cohere 10M vectors</b>, 768 dimensions) under a high filtering rate (<b>99% vectors</b>), at varying parallel levels.
+Results will show index building time, recall, and maximum QPS.""",
+    },
+]
+
+
+class InputType(IntEnum):
+    Text = 20001
+    Number = 20002
+    Option = 20003
+
+
+class CaseConfigInput(BaseModel):
+    label: CaseConfigParamType
+    inputType: InputType = InputType.Text
+    inputConfig: dict = {}
+    # todo type should be a function
+    isDisplayed: typing.Any = lambda x: True
+
+
+CaseConfigParamInput_IndexType = CaseConfigInput(
+    label=CaseConfigParamType.IndexType,
+    inputType=InputType.Option,
+    inputConfig={
+        "options": [
+            IndexType.HNSW.value,
+            IndexType.IVFFlat.value,
+            IndexType.DISKANN.value,
+            IndexType.Flat.value,
+            IndexType.AUTOINDEX.value,
+        ],
+    },
+)
+
+CaseConfigParamInput_M = CaseConfigInput(
+    label=CaseConfigParamType.M,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 4,
+        "max": 64,
+        "value": 30,
+    },
+    isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
+    == IndexType.HNSW.value,
+)
+
+CaseConfigParamInput_EFConstruction_Milvus = CaseConfigInput(
+    label=CaseConfigParamType.EFConstruction,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 8,
+        "max": 512,
+        "value": 360,
+    },
+    isDisplayed=lambda config: config[CaseConfigParamType.IndexType]
+    == IndexType.HNSW.value,
+)
+
+CaseConfigParamInput_EFConstruction_Weaviate = CaseConfigInput(
+    label=CaseConfigParamType.EFConstruction,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 8,
+        "max": 512,
+        "value": 128,
+    },
+)
+
+CaseConfigParamInput_EFConstruction_ES = CaseConfigInput(
+    label=CaseConfigParamType.EFConstruction,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 8,
+        "max": 512,
+        "value": 360,
+    },
+)
+
+CaseConfigParamInput_M_ES = CaseConfigInput(
+    label=CaseConfigParamType.M,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 4,
+        "max": 64,
+        "value": 30,
+    },
+)
+
+CaseConfigParamInput_NumCandidates_ES = CaseConfigInput(
+    label=CaseConfigParamType.numCandidates,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 1,
+        "max": 10000,
+        "value": 100,
+    },
+)
+
+CaseConfigParamInput_EF_Milvus = CaseConfigInput(
+    label=CaseConfigParamType.EF,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 100,
+        "max": MAX_STREAMLIT_INT,
+        "value": 100,
+    },
+    isDisplayed=lambda config: config[CaseConfigParamType.IndexType]
+    == IndexType.HNSW.value,
+)
+
+CaseConfigParamInput_EF_Weaviate = CaseConfigInput(
+    label=CaseConfigParamType.EF,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": -1,
+        "max": MAX_STREAMLIT_INT,
+        "value": -1,
+    },
+)
+
+CaseConfigParamInput_MaxConnections = CaseConfigInput(
+    label=CaseConfigParamType.MaxConnections,
+    inputType=InputType.Number,
+    inputConfig={"min": 1, "max": MAX_STREAMLIT_INT, "value": 64},
+)
+
+CaseConfigParamInput_SearchList = CaseConfigInput(
+    label=CaseConfigParamType.SearchList,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 100,
+        "max": MAX_STREAMLIT_INT,
+        "value": 100,
+    },
+    isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
+    == IndexType.DISKANN.value,
+)
+
+CaseConfigParamInput_Nlist = CaseConfigInput(
+    label=CaseConfigParamType.Nlist,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 1,
+        "max": 65536,
+        "value": 1000,
+    },
+    isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
+    == IndexType.IVFFlat.value,
+)
+
+CaseConfigParamInput_Nprobe = CaseConfigInput(
+    label=CaseConfigParamType.Nprobe,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 1,
+        "max": 65536,
+        "value": 10,
+    },
+    isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
+    == IndexType.IVFFlat.value,
+)
+
+
+MilvusLoadConfig = [
+    CaseConfigParamInput_IndexType,
+    CaseConfigParamInput_M,
+    CaseConfigParamInput_EFConstruction_Milvus,
+    CaseConfigParamInput_Nlist,
+]
+
+
+MilvusPerformanceConfig = [
+    CaseConfigParamInput_IndexType,
+    CaseConfigParamInput_M,
+    CaseConfigParamInput_EFConstruction_Milvus,
+    CaseConfigParamInput_EF_Milvus,
+    CaseConfigParamInput_SearchList,
+    CaseConfigParamInput_Nlist,
+    CaseConfigParamInput_Nprobe,
+]
+
+WeaviateLoadConfig = [
+    CaseConfigParamInput_MaxConnections,
+    CaseConfigParamInput_EFConstruction_Weaviate,
+]
+
+WeaviatePerformanceConfig = [
+    CaseConfigParamInput_MaxConnections,
+    CaseConfigParamInput_EFConstruction_Weaviate,
+    CaseConfigParamInput_EF_Weaviate,
+]
+
+ESLoadingConfig = [CaseConfigParamInput_EFConstruction_ES, CaseConfigParamInput_M_ES]
+
+ESPerformanceConfig = [
+    CaseConfigParamInput_EFConstruction_ES,
+    CaseConfigParamInput_M_ES,
+    CaseConfigParamInput_NumCandidates_ES,
+]
+
+CASE_CONFIG_MAP = {
+    DB.Milvus: {
+        CaseType.CapacityLDim: MilvusLoadConfig,
+        CaseType.CapacitySDim: MilvusLoadConfig,
+        CaseType.PerformanceLZero: MilvusPerformanceConfig,
+        CaseType.PerformanceMZero: MilvusPerformanceConfig,
+        CaseType.PerformanceSZero: MilvusPerformanceConfig,
+        CaseType.PerformanceLLow: MilvusPerformanceConfig,
+        CaseType.PerformanceMLow: MilvusPerformanceConfig,
+        CaseType.PerformanceSLow: MilvusPerformanceConfig,
+        CaseType.PerformanceLHigh: MilvusPerformanceConfig,
+        CaseType.PerformanceMHigh: MilvusPerformanceConfig,
+        CaseType.PerformanceSHigh: MilvusPerformanceConfig,
+        CaseType.Performance100M: MilvusPerformanceConfig,
+    },
+    DB.WeaviateCloud: {
+        CaseType.CapacityLDim: WeaviateLoadConfig,
+        CaseType.CapacitySDim: WeaviateLoadConfig,
+        CaseType.PerformanceLZero: WeaviatePerformanceConfig,
+        CaseType.PerformanceMZero: WeaviatePerformanceConfig,
+        CaseType.PerformanceSZero: WeaviatePerformanceConfig,
+        CaseType.PerformanceLLow: WeaviatePerformanceConfig,
+        CaseType.PerformanceMLow: WeaviatePerformanceConfig,
+        CaseType.PerformanceSLow: WeaviatePerformanceConfig,
+        CaseType.PerformanceLHigh: WeaviatePerformanceConfig,
+        CaseType.PerformanceMHigh: WeaviatePerformanceConfig,
+        CaseType.PerformanceSHigh: WeaviatePerformanceConfig,
+        CaseType.Performance100M: WeaviatePerformanceConfig,
+    },
+    DB.ElasticCloud: {
+        CaseType.CapacityLDim: ESLoadingConfig,
+        CaseType.CapacitySDim: ESLoadingConfig,
+        CaseType.PerformanceLZero: ESPerformanceConfig,
+        CaseType.PerformanceMZero: ESPerformanceConfig,
+        CaseType.PerformanceSZero: ESPerformanceConfig,
+        CaseType.PerformanceLLow: ESPerformanceConfig,
+        CaseType.PerformanceMLow: ESPerformanceConfig,
+        CaseType.PerformanceSLow: ESPerformanceConfig,
+        CaseType.PerformanceLHigh: ESPerformanceConfig,
+        CaseType.PerformanceMHigh: ESPerformanceConfig,
+        CaseType.PerformanceSHigh: ESPerformanceConfig,
+        CaseType.Performance100M: ESPerformanceConfig,
+    },
+}
+
+DB_DBLABEL_TO_PRICE = {
+    DB.Milvus.value: {},
+    DB.ZillizCloud.value: {
+        "1cu-perf": 0.159,
+        "8cu-perf": 1.272,
+        "1cu-cap": 0.159,
+        "2cu-cap": 0.318,
+    },
+    DB.WeaviateCloud.value: {
+        # "sandox": 0, # emmmm
+        "standard": 10.10,
+        "bus_crit": 32.60,
+    },
+    DB.ElasticCloud.value: {
+        "free-5c8g": 0.260,
+        "upTo2.5c8g": 0.4793,
+    },
+    DB.QdrantCloud.value: {
+        "0.5c4g-1node": 0.052,
+        "2c8g-1node": 0.166,
+        "4c16g-5node": 1.426,
+    },
+    DB.Pinecone.value: {
+        "s1.x1": 0.0973,
+        "s1.x2": 0.194,
+        "p1.x1": 0.0973,
+        "p1.x5-2pod": 0.973,
+        "p2.x1": 0.146,
+        "p2.x5-2pod": 1.46,
+    },
+}

vectordb_bench/frontend/pages/qps_with_price.py

@@ -0,0 +1,60 @@
+import streamlit as st
+from vectordb_bench.frontend.components.check_results.priceTable import priceTable
+from vectordb_bench.frontend.const import *
+from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
+from vectordb_bench.frontend.components.check_results.nav import NavToResults, NavToRunTest
+from vectordb_bench.frontend.components.check_results.charts import drawMetricChart
+from vectordb_bench.frontend.components.check_results.filters import getshownData
+from vectordb_bench.interface import benchMarkRunner
+
+
+def main():
+    st.set_page_config(
+        page_title="VectorDB Benchmark",
+        page_icon="https://assets.zilliz.com/favicon_f7f922fe27.png",
+        # layout="wide",
+        # initial_sidebar_state="collapsed",
+    )
+
+    # header
+    drawHeaderIcon(st)
+
+    allResults = benchMarkRunner.get_results()
+
+    st.title("Vector DB Benchmark (QPS / Price)")
+    st.subheader("Price List")
+
+    # results selector
+    resultSelectorContainer = st.sidebar.container()
+    shownData, failedTasks, showCases = getshownData(allResults, resultSelectorContainer)
+
+    resultSelectorContainer.divider()
+
+    # nav
+    navContainer = st.sidebar.container()
+    NavToRunTest(navContainer)
+    NavToResults(navContainer)
+    
+    # price table
+    priceTableContainer = st.container()
+    priceMap = priceTable(priceTableContainer, shownData)
+
+    # charts
+    for case in showCases:
+        chartContainer = st.container()
+        data = [data for data in shownData if data["case"] == case]
+        dataWithMetric = []
+        metric = "qps_per_dollar (qps / price)"
+        for d in data:
+            qps = d.get("qps", 0)
+            price = priceMap.get(d["db"], {}).get(d["db_label"], 0)
+            if qps > 0 and price > 0:
+                d[metric] = d["qps"] / price
+                dataWithMetric.append(d)
+        if len(dataWithMetric) > 0:
+            chartContainer.subheader(case)
+            drawMetricChart(data, metric, chartContainer)
+
+
+if __name__ == "__main__":
+    main()

vectordb_bench/frontend/pages/run_test.py

@@ -0,0 +1,59 @@
+import streamlit as st
+from vectordb_bench.frontend.components.run_test.autoRefresh import autoRefresh
+from vectordb_bench.frontend.components.run_test.caseSelector import caseSelector
+from vectordb_bench.frontend.components.run_test.dbConfigSetting import dbConfigSettings
+from vectordb_bench.frontend.components.run_test.dbSelector import dbSelector
+from vectordb_bench.frontend.components.run_test.generateTasks import generate_tasks
+from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
+from vectordb_bench.frontend.components.run_test.hideSidebar import hideSidebar
+from vectordb_bench.frontend.components.check_results.nav import NavToResults
+from vectordb_bench.frontend.components.run_test.submitTask import submitTask
+
+
+def main():
+    st.set_page_config(
+        page_title="VectorDB Benchmark",
+        page_icon="https://assets.zilliz.com/favicon_f7f922fe27.png",
+        # layout="wide",
+        initial_sidebar_state="collapsed",
+    )
+    # header
+    drawHeaderIcon(st)
+
+    # hide sidebar
+    hideSidebar(st)
+
+    # nav to results
+    NavToResults(st)
+
+    # header
+    st.title("Run Your Test")
+    # st.write("description [todo]")
+
+    # select db
+    dbSelectorContainer = st.container()
+    activedDbList = dbSelector(dbSelectorContainer)
+
+    # db config setting
+    dbConfigs = {}
+    if len(activedDbList) > 0:
+        dbConfigContainer = st.container()
+        dbConfigs = dbConfigSettings(dbConfigContainer, activedDbList)
+
+    # select case and set db_case_config
+    caseSelectorContainer = st.container()
+    activedCaseList, allCaseConfigs = caseSelector(caseSelectorContainer, activedDbList)
+
+    # generate tasks
+    tasks = generate_tasks(activedDbList, dbConfigs, activedCaseList, allCaseConfigs)
+
+    # submit
+    submitContainer = st.container()
+    submitTask(submitContainer, tasks)
+
+    # autofresh
+    autoRefresh()
+
+
+if __name__ == "__main__":
+    main()

vectordb_bench/frontend/utils.py

@@ -0,0 +1,6 @@
+from vectordb_bench.models import CaseType
+
+passwordKeys = ["password", "api_key"]
+def inputIsPassword(key: str) -> bool:
+    return key.lower() in passwordKeys
+

vectordb_bench/frontend/vdb_benchmark.py

@@ -0,0 +1,42 @@
+import streamlit as st
+from vectordb_bench.frontend.const import *
+from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
+from vectordb_bench.frontend.components.check_results.nav import NavToQPSWithPrice, NavToRunTest
+from vectordb_bench.frontend.components.check_results.charts import drawCharts
+from vectordb_bench.frontend.components.check_results.filters import getshownData
+from vectordb_bench.interface import benchMarkRunner
+
+
+def main():
+    st.set_page_config(
+        page_title="VectorDB Benchmark",
+        page_icon="https://assets.zilliz.com/favicon_f7f922fe27.png",
+        # layout="wide",
+        # initial_sidebar_state="collapsed",
+    )
+
+    # header
+    drawHeaderIcon(st)
+
+    allResults = benchMarkRunner.get_results()
+
+    st.title("Vector Database Benchmark")
+    # st.write("description [todo]")
+
+    # results selector
+    resultSelectorContainer = st.sidebar.container()
+    shownData, failedTasks, showCases = getshownData(allResults, resultSelectorContainer)
+
+    resultSelectorContainer.divider()
+
+    # nav
+    navContainer = st.sidebar.container()
+    NavToRunTest(navContainer)
+    NavToQPSWithPrice(navContainer)
+
+    # charts
+    drawCharts(st, shownData, failedTasks, showCases)
+
+
+if __name__ == "__main__":
+    main()