vectordb-bench 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

vectordb_bench/frontend/pages/run_test.py

@@ -4,19 +4,17 @@ from vectordb_bench.frontend.components.run_test.caseSelector import caseSelecto
 from vectordb_bench.frontend.components.run_test.dbConfigSetting import dbConfigSettings
 from vectordb_bench.frontend.components.run_test.dbSelector import dbSelector
 from vectordb_bench.frontend.components.run_test.generateTasks import generate_tasks
-from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
 from vectordb_bench.frontend.components.run_test.hideSidebar import hideSidebar
-from vectordb_bench.frontend.components.check_results.nav import NavToResults
 from vectordb_bench.frontend.components.run_test.submitTask import submitTask
+from vectordb_bench.frontend.components.check_results.nav import NavToResults
+from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
+from vectordb_bench.frontend.components.check_results.stPageConfig import initRunTestPageConfig
 
 
 def main():
-    st.set_page_config(
-        page_title="VectorDB Benchmark",
-        page_icon="https://assets.zilliz.com/favicon_f7f922fe27.png",
-        # layout="wide",
-        initial_sidebar_state="collapsed",
-    )
+    # set page config
+    initRunTestPageConfig(st)
+    
     # header
     drawHeaderIcon(st)
 
@@ -36,20 +34,28 @@ def main():
 
     # db config setting
     dbConfigs = {}
+    isAllValid = True
     if len(activedDbList) > 0:
         dbConfigContainer = st.container()
-        dbConfigs = dbConfigSettings(dbConfigContainer, activedDbList)
+        dbConfigs, isAllValid = dbConfigSettings(dbConfigContainer, activedDbList)
 
     # select case and set db_case_config
     caseSelectorContainer = st.container()
     activedCaseList, allCaseConfigs = caseSelector(caseSelectorContainer, activedDbList)
 
     # generate tasks
-    tasks = generate_tasks(activedDbList, dbConfigs, activedCaseList, allCaseConfigs)
+    tasks = (
+        generate_tasks(activedDbList, dbConfigs, activedCaseList, allCaseConfigs)
+        if isAllValid
+        else []
+    )
 
     # submit
     submitContainer = st.container()
-    submitTask(submitContainer, tasks)
+    submitTask(submitContainer, tasks, isAllValid)
+
+    # nav to results
+    NavToResults(st, key="footer-nav-to-results")
 
     # autofresh
     autoRefresh()

vectordb_bench/frontend/vdb_benchmark.py

@@ -1,19 +1,18 @@
 import streamlit as st
-from vectordb_bench.frontend.const import *
+from vectordb_bench.frontend.components.check_results.footer import footer
+from vectordb_bench.frontend.components.check_results.stPageConfig import initResultsPageConfig
 from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
-from vectordb_bench.frontend.components.check_results.nav import NavToQPSWithPrice, NavToRunTest
+from vectordb_bench.frontend.components.check_results.nav import NavToQuriesPerDollar, NavToRunTest
 from vectordb_bench.frontend.components.check_results.charts import drawCharts
 from vectordb_bench.frontend.components.check_results.filters import getshownData
+from vectordb_bench.frontend.components.get_results.saveAsImage import getResults
+from vectordb_bench.frontend.const.styles import *
 from vectordb_bench.interface import benchMarkRunner
 
 
 def main():
-    st.set_page_config(
-        page_title="VectorDB Benchmark",
-        page_icon="https://assets.zilliz.com/favicon_f7f922fe27.png",
-        # layout="wide",
-        # initial_sidebar_state="collapsed",
-    )
+    # set page config
+    initResultsPageConfig(st)
 
     # header
     drawHeaderIcon(st)
@@ -21,22 +20,30 @@ def main():
     allResults = benchMarkRunner.get_results()
 
     st.title("Vector Database Benchmark")
-    # st.write("description [todo]")
 
-    # results selector
+    # results selector and filter
     resultSelectorContainer = st.sidebar.container()
-    shownData, failedTasks, showCases = getshownData(allResults, resultSelectorContainer)
+    shownData, failedTasks, showCases = getshownData(
+        allResults, resultSelectorContainer
+    )
 
     resultSelectorContainer.divider()
 
     # nav
     navContainer = st.sidebar.container()
     NavToRunTest(navContainer)
-    NavToQPSWithPrice(navContainer)
+    NavToQuriesPerDollar(navContainer)
+
+    # save or share
+    resultesContainer = st.sidebar.container()
+    getResults(resultesContainer, "vectordb_bench")
 
     # charts
     drawCharts(st, shownData, failedTasks, showCases)
 
+    # footer
+    footer(st.container())
+
 
 if __name__ == "__main__":
     main()

vectordb_bench/metric.py

@@ -20,23 +20,32 @@ class Metric:
     serial_latency_p99: float = 0.0
     recall: float = 0.0
 
+
+QURIES_PER_DOLLAR_METRIC = "QP$ (Quries per Dollar)"
+LOAD_DURATION_METRIC = "load_duration"
+SERIAL_LATENCY_P99_METRIC = "serial_latency_p99"
+MAX_LOAD_COUNT_METRIC = "max_load_count"
+QPS_METRIC = "qps"
+RECALL_METRIC = "recall"
+
 metricUnitMap = {
-    'load_duration': 's',
-    'serial_latency_p99': 'ms',
-    'max_load_count': 'K'
+    LOAD_DURATION_METRIC: "s",
+    SERIAL_LATENCY_P99_METRIC: "ms",
+    MAX_LOAD_COUNT_METRIC: "K",
+    QURIES_PER_DOLLAR_METRIC: "K",
 }
 
 lowerIsBetterMetricList = [
-    "load_duration",
-    "serial_latency_p99",
+    LOAD_DURATION_METRIC,
+    SERIAL_LATENCY_P99_METRIC,
 ]
 
 metricOrder = [
-    "qps",
-    "recall",
-    "load_duration",
-    "serial_latency_p99",
-    "max_load_count",
+    QPS_METRIC,
+    RECALL_METRIC,
+    LOAD_DURATION_METRIC,
+    SERIAL_LATENCY_P99_METRIC,
+    MAX_LOAD_COUNT_METRIC,
 ]
 
 

vectordb_bench/models.py

@@ -12,6 +12,7 @@ from .backend.clients import (
     DBCaseConfig,
     IndexType,
 )
+from .backend.cases import CaseType
 from .base import BaseModel
 from . import config
 from .metric import Metric
@@ -27,38 +28,6 @@ class PerformanceTimeoutError(TimeoutError):
     pass
 
 
-class CaseType(Enum):
-    """
-    Value will be displayed in UI
-    """
-
-    CapacitySDim = "Capacity Test (Large-dim)"
-    CapacityLDim = "Capacity Test (Small-dim)"
-
-    Performance100M = "Search Performance Test (XLarge Dataset)"
-    PerformanceLZero = "Search Performance Test (Large Dataset)"
-    PerformanceMZero = "Search Performance Test (Medium Dataset)"
-    PerformanceSZero = "Search Performance Test (Small Dataset)"
-
-    PerformanceLLow = (
-        "Filtering Search Performance Test (Large Dataset, Low Filtering Rate)"
-    )
-    PerformanceMLow = (
-        "Filtering Search Performance Test (Medium Dataset, Low Filtering Rate)"
-    )
-    PerformanceSLow = (
-        "Filtering Search Performance Test (Small Dataset, Low Filtering Rate)"
-    )
-    PerformanceLHigh = (
-        "Filtering Search Performance Test (Large Dataset, High Filtering Rate)"
-    )
-    PerformanceMHigh = (
-        "Filtering Search Performance Test (Medium Dataset, High Filtering Rate)"
-    )
-    PerformanceSHigh = (
-        "Filtering Search Performance Test (Small Dataset, High Filtering Rate)"
-    )
-
 
 class CaseConfigParamType(Enum):
     """
@@ -74,6 +43,8 @@ class CaseConfigParamType(Enum):
     Nprobe = "nprobe"
     MaxConnections = "maxConnections"
     numCandidates = "num_candidates"
+    lists = "lists"
+    probes = "probes"
 
 
 class CustomizedCase(BaseModel):
@@ -151,6 +122,7 @@ class TestResult(BaseModel):
                 task_config = case_result.get("task_config")
                 db = DB(task_config.get("db"))
                 dbcls = db.init_cls
+
                 task_config["db_config"] = dbcls.config_cls()(
                     **task_config["db_config"]
                 )
@@ -197,20 +169,21 @@ class TestResult(BaseModel):
         max_qps = max(map(len, [str(f.metrics.qps) for f in filtered_results])) + 3
         max_recall = max(map(len, [str(f.metrics.recall) for f in filtered_results])) + 3
 
-        max_db_labels = 8 if max_db_labels == 0 else max_db_labels
-        max_load_dur = 11 if max_load_dur == 0 else max_load_dur + 3
-        max_qps = 10 if max_qps == 0 else max_load_dur + 3
-        max_recall = 13 if max_recall == 0 else max_recall + 3
+        max_db_labels = 8 if max_db_labels < 8 else max_db_labels
+        max_load_dur = 11 if max_load_dur < 11 else max_load_dur
+        max_qps = 10 if max_qps < 10 else max_qps
+        max_recall = 13 if max_recall < 13 else max_recall
 
-        LENGTH = (max_db, max_db_labels, max_case, len(self.task_label), max_load_dur, max_qps, 15, max_recall, 14)
+        LENGTH = (max_db, max_db_labels, max_case, len(self.task_label), max_load_dur, max_qps, 15, max_recall, 14, 5)
 
         DATA_FORMAT = (
-            f"%-{max_db}s | %-{max_db_labels}s %-{max_case}s %-{len(self.task_label)}s "
-            f"| %-{max_load_dur}s %-{max_qps}s %-15s %-{max_recall}s %-14s"
+            f"%-{max_db}s | %-{max_db_labels}s %-{max_case}s %-{len(self.task_label)}s"
+            f" | %-{max_load_dur}s %-{max_qps}s %-15s %-{max_recall}s %-14s"
+            f" | %-5s"
         )
 
         TITLE = DATA_FORMAT % (
-            "DB", "db_label", "case", "label", "load_dur", "qps", "latency(p99)", "recall", "max_load_count")
+            "DB", "db_label", "case", "label", "load_dur", "qps", "latency(p99)", "recall", "max_load_count", "label")
         SPLIT = DATA_FORMAT%tuple(map(lambda x:"-"*x, LENGTH))
         SUMMERY_FORMAT = ("Task summery: run_id=%s, task_label=%s") % (self.run_id[:5], self.task_label)
         fmt = [SUMMERY_FORMAT, TITLE, SPLIT]
@@ -227,6 +200,7 @@ class TestResult(BaseModel):
                 f.metrics.serial_latency_p99,
                 f.metrics.recall,
                 f.metrics.max_load_count,
+                f.label.value,
             ))
 
         tmp_logger = logging.getLogger("no_color")

vectordb_bench/results/dbPrices.json

@@ -0,0 +1,32 @@
+{
+  "Milvus": {},
+  "ZillizCloud": {
+    "1cu-perf": 0.159,
+    "8cu-perf": 1.272,
+    "1cu-cap": 0.159,
+    "2cu-cap": 0.318
+  },
+  "WeaviateCloud": {
+    "standard": 10.1,
+    "bus_crit": 32.6
+  },
+  "ElasticCloud": {
+    "upTo2.5c8g": 0.4793
+  },
+  "QdrantCloud": {
+    "0.5c4g-1node": 0.052,
+    "2c8g-1node": 0.166,
+    "4c16g-1node": 0.2852,
+    "4c16g-5node": 1.426
+  },
+  "Pinecone": {
+    "s1.x1": 0.0973,
+    "s1.x2": 0.194,
+    "p1.x1": 0.0973,
+    "p2.x1": 0.146,
+    "p2.x1-8node": 1.168,
+    "p1.x1-8node": 0.779,
+    "s1.x1-2node": 0.195
+  },
+  "PgVector": {}
+}

vectordb_bench/results/getLeaderboardData.py

@@ -0,0 +1,52 @@
+from vectordb_bench import config
+import ujson
+import pathlib
+from vectordb_bench.backend.cases import CaseType
+from vectordb_bench.frontend.const.dbPrices import DB_DBLABEL_TO_PRICE
+from vectordb_bench.interface import benchMarkRunner
+from vectordb_bench.models import CaseResult, ResultLabel, TestResult
+
+taskLabelToCode = {
+    ResultLabel.FAILED: -1,
+    ResultLabel.OUTOFRANGE: -2,
+    ResultLabel.NORMAL: 1,
+}
+
+
+def main():
+    allResults: list[TestResult] = benchMarkRunner.get_results()
+    results: list[CaseResult] = []
+    for result in allResults:
+        if result.task_label == "standard":
+            results += result.results
+
+    if allResults is not None:
+        data = [
+            {
+                "db": d.task_config.db.value,
+                "db_label": d.task_config.db_config.db_label,
+                "db_name": d.task_config.db_name,
+                "case": d.task_config.case_config.case_id.case_name,
+                "qps": d.metrics.qps,
+                "latency": d.metrics.serial_latency_p99,
+                "label": taskLabelToCode[d.label],
+            }
+            for d in results
+            if d.task_config.case_config.case_id != CaseType.CapacityDim128
+            and d.task_config.case_config.case_id != CaseType.CapacityDim960
+        ]
+
+        # compute qp$
+        for d in data:
+            db = d["db"]
+            db_label = d["db_label"]
+            qps = d["qps"]
+            price = DB_DBLABEL_TO_PRICE.get(db, {}).get(db_label, 0)
+            d["qp$"] = qps / price if price > 0 else 0.0
+
+        with open(pathlib.Path(config.RESULTS_LOCAL_DIR, "leaderboard.json"), "w") as f:
+            ujson.dump(data, f)
+
+
+if __name__ == "__main__":
+    main()

vectordb_bench/results/leaderboard.json

@@ -0,0 +1 @@
+[{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"label":1,"qp$":31.76903818068016},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"label":1,"qp$":31.660546630502814},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"label":1,"qp$":57.621740037554765},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-2,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"label":1,"qp$":2075.5622641509435},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"label":1,"qp$":1708.5440251572327},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"label":1,"qp$":1361.777358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"label":1,"qp$":389.79654088050313},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"label":1,"qp$":185.99968553459118},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"label":1,"qp$":128.92767295597486},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"label":1,"qp$":1823.71572327044},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"label":1,"qp$":1337.2732704402515},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"label":1,"qp$":1248.5971698113208},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"label":1,"qp$":2.083193251533742},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"label":1,"qp$":0.02342331288343558},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"label":1,"qp$":0.9815950920245399},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"label":1,"qp$":6.251138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"label":1,"qp$":0.07437623762376237},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"label":1,"qp$":2.983742574257426},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"label":1,"qp$":3246.9811320754716},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"label":1,"qp$":2231.708176100629},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"label":1,"qp$":2688.8232704402517},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"label":1,"qp$":2267.837264150943},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"label":1,"qp$":1328.2860849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"label":1,"qp$":1193.1440251572328},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"label":1,"qp$":646.6444968553459},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"label":1,"qp$":297.8888364779874},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"label":1,"qp$":171.92248427672953},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"label":1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"label":1,"qp$":376.9267180925666},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"label":1,"qp$":260.9022440392707},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"label":1,"qp$":810.8624123422161},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"label":1,"qp$":77.31276297335204},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"label":1,"qp$":61.1922159887798},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"label":1,"qp$":88.20799438990183},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"label":1,"qp$":844.0424263674614},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"label":1,"qp$":664.2352734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"label":1,"qp$":1099.2692847124824},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"label":1,"qp$":192.84069886947586},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"label":1,"qp$":188.7142857142857},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"label":1,"qp$":259.7574511819116},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"label":1,"qp$":1793.13698630137},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"label":1,"qp$":1138.2541095890413},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"label":1,"qp$":833.677397260274},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"label":1,"qp$":479.1253854059609},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"label":1,"qp$":436.6454265159301},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"label":1,"qp$":1428.0359712230218},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"label":1,"qp$":106.37794871794871},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"label":1,"qp$":104.09897435897435},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"label":1,"qp$":135.75333333333333},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"label":1,"qp$":44.44871794871795},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"label":1,"qp$":40.06205128205128},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"label":1,"qp$":86.50769230769231},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"label":1,"qp$":312.57148972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"label":1,"qp$":278.7047089041096},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"label":1,"qp$":510.95393835616443},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"label":1,"qp$":259.5928082191781},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"label":1,"qp$":116.46789383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"label":1,"qp$":57.125085616438355},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"label":1,"qp$":168.49152759948652},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"label":1,"qp$":164.2281129653402},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"label":1,"qp$":764.8860077021822},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0}]