vectordb-bench 1.0.4__py3-none-any.whl → 1.0.7__py3-none-any.whl

vectordb_bench/results/getLeaderboardDataV2.py

@@ -2,10 +2,11 @@ import json
 import logging
 
 
-from vectordb_bench.backend.cases import CaseType
+from vectordb_bench.backend.cases import CaseType, StreamingPerformanceCase
 from vectordb_bench.backend.clients import DB
 from vectordb_bench.models import CaseResult
 from vectordb_bench import config
+import numpy as np
 
 logging.basicConfig(level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s")
 
@@ -29,9 +30,11 @@ def save_to_json(data: list[dict], file_name: str):
 def main():
     standard_2025_case_results = get_standard_2025_results()
     data = []
+    streaming_data = []
     for case_result in standard_2025_case_results:
         db = case_result.task_config.db
         label = case_result.task_config.db_config.db_label
+        db_name = f"{db.value}{f'-{label}' if label else ''}"
         metrics = case_result.metrics
         qps = metrics.qps
         latency = metrics.serial_latency_p99
@@ -45,14 +48,32 @@ def main():
                     "dataset": dataset,
                     "db": db.value,
                     "label": label,
-                    "db_name": f"{db.value}-{label}",
+                    "db_name": db_name,
                     "qps": round(qps, 4),
                     "latency": round(latency, 4),
                     "recall": round(recall, 4),
                     "filter_ratio": round(filter_ratio, 3),
                 }
             )
+        else:
+            case: StreamingPerformanceCase = case
+            # use 90p search stage results to represent streaming performance
+            qps_90p = metrics.st_max_qps_list_list[metrics.st_search_stage_list.index(90)]
+            latency_90p = metrics.st_serial_latency_p99_list[metrics.st_search_stage_list.index(90)]
+            insert_rate = case.insert_rate
+            streaming_data.append(
+                {
+                    "dataset": dataset,
+                    "db": db.value,
+                    "label": label,
+                    "db_name": db_name,
+                    "insert_rate": insert_rate,
+                    "streaming_qps": round(qps_90p, 4),
+                    "streaming_latency": round(latency_90p, 4),
+                }
+            )
     save_to_json(data, config.RESULTS_LOCAL_DIR / "leaderboard_v2.json")
+    save_to_json(streaming_data, config.RESULTS_LOCAL_DIR / "leaderboard_v2_streaming.json")
 
 
 if __name__ == "__main__":

vectordb_bench/results/leaderboard_v2.json

@@ -2658,5 +2658,205 @@
         "latency": 3.3,
         "recall": 0.9147,
         "filter_ratio": 0.5
+    },
+    {
+        "dataset": "Cohere (Medium)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 199.4972,
+        "latency": 337.1,
+        "recall": 0.8717,
+        "filter_ratio": 0.0
+    },
+    {
+        "dataset": "Cohere (Medium)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 192.1164,
+        "latency": 345.6,
+        "recall": 0.4276,
+        "filter_ratio": 0.999
+    },
+    {
+        "dataset": "Cohere (Medium)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 197.4455,
+        "latency": 349.3,
+        "recall": 0.5314,
+        "filter_ratio": 0.998
+    },
+    {
+        "dataset": "Cohere (Medium)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 196.9391,
+        "latency": 263.4,
+        "recall": 0.6549,
+        "filter_ratio": 0.995
+    },
+    {
+        "dataset": "Cohere (Medium)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 201.5401,
+        "latency": 282.4,
+        "recall": 0.7086,
+        "filter_ratio": 0.99
+    },
+    {
+        "dataset": "Cohere (Medium)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 202.2424,
+        "latency": 301.7,
+        "recall": 0.7592,
+        "filter_ratio": 0.98
+    },
+    {
+        "dataset": "Cohere (Medium)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 198.599,
+        "latency": 358.8,
+        "recall": 0.8085,
+        "filter_ratio": 0.95
+    },
+    {
+        "dataset": "Cohere (Medium)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 199.0349,
+        "latency": 275.3,
+        "recall": 0.8325,
+        "filter_ratio": 0.9
+    },
+    {
+        "dataset": "Cohere (Medium)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 202.1405,
+        "latency": 282.6,
+        "recall": 0.8492,
+        "filter_ratio": 0.8
+    },
+    {
+        "dataset": "Cohere (Medium)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 201.1282,
+        "latency": 269.2,
+        "recall": 0.8637,
+        "filter_ratio": 0.5
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 194.8021,
+        "latency": 559.8,
+        "recall": 0.86,
+        "filter_ratio": 0.0
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 187.4268,
+        "latency": 453.7,
+        "recall": 0.4692,
+        "filter_ratio": 0.999
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 198.397,
+        "latency": 506.9,
+        "recall": 0.5409,
+        "filter_ratio": 0.998
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 174.3549,
+        "latency": 496.9,
+        "recall": 0.6279,
+        "filter_ratio": 0.995
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 172.95,
+        "latency": 515.6,
+        "recall": 0.7004,
+        "filter_ratio": 0.99
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 190.9747,
+        "latency": 517.4,
+        "recall": 0.7398,
+        "filter_ratio": 0.98
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 186.0237,
+        "latency": 474.0,
+        "recall": 0.7847,
+        "filter_ratio": 0.95
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 192.1458,
+        "latency": 480.5,
+        "recall": 0.8103,
+        "filter_ratio": 0.9
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 179.4203,
+        "latency": 497.5,
+        "recall": 0.8273,
+        "filter_ratio": 0.8
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "qps": 199.5444,
+        "latency": 463.9,
+        "recall": 0.8478,
+        "filter_ratio": 0.5
     }
 ]

vectordb_bench/results/leaderboard_v2_streaming.json

@@ -0,0 +1,128 @@
+[
+    {
+        "dataset": "Cohere (Large)",
+        "db": "ElasticCloud",
+        "label": "8c60g",
+        "db_name": "ElasticCloud-8c60g",
+        "insert_rate": 500,
+        "streaming_qps": 61.6708,
+        "streaming_latency": 0.0794
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "ElasticCloud",
+        "label": "8c60g",
+        "db_name": "ElasticCloud-8c60g",
+        "insert_rate": 1000,
+        "streaming_qps": 61.8172,
+        "streaming_latency": 0.2223
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "Milvus",
+        "label": "16c64g-sq8",
+        "db_name": "Milvus-16c64g-sq8",
+        "insert_rate": 500,
+        "streaming_qps": 305.9971,
+        "streaming_latency": 0.005
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "Milvus",
+        "label": "16c64g-sq8",
+        "db_name": "Milvus-16c64g-sq8",
+        "insert_rate": 1000,
+        "streaming_qps": 155.9613,
+        "streaming_latency": 0.0203
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "Pinecone",
+        "label": "p2.x8-1node",
+        "db_name": "Pinecone-p2.x8-1node",
+        "insert_rate": 500,
+        "streaming_qps": 367.4299,
+        "streaming_latency": 1.8286
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "Pinecone",
+        "label": "p2.x8-1node",
+        "db_name": "Pinecone-p2.x8-1node",
+        "insert_rate": 1000,
+        "streaming_qps": 369.6771,
+        "streaming_latency": 5.992
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "QdrantCloud",
+        "label": "16c64g",
+        "db_name": "QdrantCloud-16c64g",
+        "insert_rate": 500,
+        "streaming_qps": 393.753,
+        "streaming_latency": 0.0162
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "QdrantCloud",
+        "label": "16c64g",
+        "db_name": "QdrantCloud-16c64g",
+        "insert_rate": 1000,
+        "streaming_qps": 347.5774,
+        "streaming_latency": 0.0118
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "OpenSearch",
+        "label": "16c128g",
+        "db_name": "OpenSearch-16c128g",
+        "insert_rate": 1000,
+        "streaming_qps": 149.7168,
+        "streaming_latency": 0.098
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "OpenSearch",
+        "label": "16c128g",
+        "db_name": "OpenSearch-16c128g",
+        "insert_rate": 500,
+        "streaming_qps": 161.6694,
+        "streaming_latency": 0.052
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "ZillizCloud",
+        "label": "8cu-perf",
+        "db_name": "ZillizCloud-8cu-perf",
+        "insert_rate": 500,
+        "streaming_qps": 2118.7516,
+        "streaming_latency": 0.0068
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "ZillizCloud",
+        "label": "8cu-perf",
+        "db_name": "ZillizCloud-8cu-perf",
+        "insert_rate": 1000,
+        "streaming_qps": 1860.2575,
+        "streaming_latency": 0.0101
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "insert_rate": 500,
+        "streaming_qps": 180.9549,
+        "streaming_latency": 0.4204
+    },
+    {
+        "dataset": "Cohere (Large)",
+        "db": "S3Vectors",
+        "label": "",
+        "db_name": "S3Vectors",
+        "insert_rate": 1000,
+        "streaming_qps": 167.2689,
+        "streaming_latency": 0.5048
+    }
+]

{vectordb_bench-1.0.4.dist-info → vectordb_bench-1.0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vectordb-bench
-Version: 1.0.4
+Version: 1.0.7
 Summary: VectorDBBench is not just an offering of benchmark results for mainstream vector databases and cloud services, it's your go-to tool for the ultimate performance and cost-effectiveness comparison. Designed with ease-of-use in mind, VectorDBBench is devised to help users, even non-professionals, reproduce results or test new systems, making the hunt for the optimal choice amongst a plethora of cloud services and open-source vector databases a breeze.
 Author-email: XuanYang-cn <xuan.yang@zilliz.com>
 Project-URL: repository, https://github.com/zilliztech/VectorDBBench
@@ -153,6 +153,7 @@ All the database client supported
 | tidb                     | `pip install vectordb-bench[tidb]`          |
 | vespa                    | `pip install vectordb-bench[vespa]`         |
 | oceanbase                | `pip install vectordb-bench[oceanbase]`     |
+| hologres                 | `pip install vectordb-bench[hologres]`      |
 
 ### Run
 
@@ -269,7 +270,7 @@ vectordbbench awsopensearch --db-label awsopensearch \
 --m 16 --ef-construction 256 \
 --host search-vector-db-prod-h4f6m4of6x7yp2rz7gdmots7w4.us-west-2.es.amazonaws.com --port 443 \
 --user vector --password '<password>' \
---case-type Performance1536D5M --num-insert-workers 10  \
+--case-type Performance1536D5M --number-of-indexing-clients 10  \
 --skip-load --num-concurrency 75
 ```
 
@@ -297,7 +298,7 @@ Options:
   --force-merge-enabled BOOLEAN   Whether to perform force merge operation
   --flush-threshold-size TEXT     Size threshold for flushing the transaction
                                   log
-  --engine TEXT                   type of engine to use valid values [faiss, lucene]
+  --engine TEXT                   type of engine to use valid values [faiss, lucene, s3vector]
   # Memory Management
   --cb-threshold TEXT             k-NN Memory circuit breaker threshold
   
@@ -372,6 +373,37 @@ Options:
   --help                          Show this message and exit.                       Show this message and exit.
   ```
 
+### Run Hologres from command line
+
+Execute tests for the index types: HGraph.
+
+```shell
+vectordbbench hologreshgraph --host xxx --port xxx --user ACCESS_ID --password ACCESS_KEY --database test \
+--m 64 --ef-construction 400 --case-type Performance768D1M \
+--index-type HGraph --ef-search 51 --k 10
+```
+
+To list the options for Hologres, execute `vectordbbench hologreshgraph --help`, The following are some Hologres-specific command-line options.
+
+```text
+$ vectordbbench hologreshgraph --help
+Usage: vectordbbench hologreshgraph [OPTIONS]
+
+Options:
+  [...]
+  --host TEXT                     Hologres host
+  --user TEXT                     Hologres username  [required]
+  --password TEXT                 Hologres database password
+  --database TEXT                 Hologres database name  [required]
+  --port INTEGER                  Hologres port  [required]
+  --m INTEGER                     hnsw m  [required]
+  --ef-construction INTEGER       hnsw ef-construction  [required]
+  --ef-search INTEGER             hnsw ef-search  [required]
+  --index-type [HGraph]           Type of index to use. Supported values:
+                                  HGraph [required]
+  --help                          Show this message and exit.
+  ```
+
 #### Using a configuration file.
 
 The vectordbbench command can optionally read some or all the options from a yaml formatted configuration file.
@@ -525,9 +557,9 @@ All standard benchmark results are generated by a client running on an 8 core, 3
 2. The next step is to select the test cases you want to perform. You can select multiple cases at once, and a form to collect corresponding parameters will appear.
 3. Finally, you'll need to provide a task label to distinguish different test results. Using the same label for different tests will result in the previous results being overwritten.
 Now we can only run one task at the same time. 
-![image](fig/run_test_select_db.png)
-![image](fig/run_test_select_case.png)
-![image](fig/run_test_submit.png)
+![image](vectordb_bench/fig/run_test_select_db.png)
+![image](vectordb_bench/fig/run_test_select_case.png)
+![image](vectordb_bench/fig/run_test_submit.png)
 
 
 ## Module
@@ -557,8 +589,8 @@ Each case provides an in-depth examination of a vector database's abilities, pro
 
 Through the `/custom` page, users can customize their own performance case using local datasets. After saving, the corresponding case can be selected from the `/run_test` page to perform the test.
 
-![image](fig/custom_dataset.png)
-![image](fig/custom_case_run_test.png)
+![image](vectordb_bench/fig/custom_dataset.png)
+![image](vectordb_bench/fig/custom_case_run_test.png)
 
 We have strict requirements for the data set format, please follow them.
 - `Folder Path` - The path to the folder containing all the files. Please ensure that all files in the folder are in the `Parquet` format.