vectordb-bench 0.0.1__tar.gz → 0.0.3__tar.gz

vectordb-bench-0.0.3/.env.example

@@ -0,0 +1,11 @@
+# LOG_LEVEL=
+# LOG_PATH=
+# LOG_NAME=
+# TIMEZONE=
+
+# NUM_PER_BATCH=
+# DEFAULT_DATASET_URL=
+
+DATASET_LOCAL_DIR="/tmp/vector_db_bench/dataset"
+
+# DROP_OLD = True

vectordb-bench-0.0.3/.github/workflows/publish_package_on_release.yml

@@ -0,0 +1,46 @@
+name: Publish Python 🐍 distributions 📦 to TestPyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build-n-publish:
+    name: Build and publish Python 🐍 distributions 📦 to PyPI
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check out from Git
+      uses: actions/checkout@v3
+    - name: Get history and tags for SCM versioning
+      run: |
+          git fetch --prune --unshallow
+          git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.11
+    - name: Install pypa/build
+      run: >-
+        python -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: >-
+        python -m
+        build
+        --sdist
+        --wheel
+        --outdir dist/
+        .
+    - name: Publish distribution 📦 to Test PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+        repository-url: https://test.pypi.org/legacy/
+    - name: Publish distribution 📦 to PyPI
+      if: startsWith(github.ref, 'refs/tags')
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.PYPI_API_TOKEN }}

vectordb-bench-0.0.3/.gitignore

@@ -0,0 +1,11 @@
+*.sw[op]
+*.egg-info
+dist/
+__pycache__
+.env
+.data/
+__MACOSX
+.DS_Store
+build/
+venv/
+.idea/

vectordb-bench-0.0.3/.ruff.toml

@@ -0,0 +1,49 @@
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+# Enable flake8-bugbear (`B`) rules.
+select = ["E", "F", "B"]
+ignore = [
+    "E501", # (line length violations)
+]
+
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM",
+    "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT",
+    "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT",
+]
+unfixable = []
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+    "__pycache__",
+    "__init__.py",
+]
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+# Assume Python 3.11.
+target-version = "py311"
+
+[mccabe]
+# Unlike Flake8, default to a complexity level of 10.
+max-complexity = 10

{vectordb-bench-0.0.1/vectordb_bench.egg-info → vectordb-bench-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vectordb-bench
-Version: 0.0.1
+Version: 0.0.3
 Summary: VectorDBBench is not just an offering of benchmark results for mainstream vector databases and cloud services, it's your go-to tool for the ultimate performance and cost-effectiveness comparison. Designed with ease-of-use in mind, VectorDBBench is devised to help users, even non-professionals, reproduce results or test new systems, making the hunt for the optimal choice amongst a plethora of cloud services and open-source vector databases a breeze.
 Author-email: XuanYang-cn <xuan.yang@zilliz.com>
 Project-URL: repository, https://github.com/zilliztech/VectorDBBench
@@ -14,6 +14,10 @@ License-File: LICENSE
 
 # VectorDBBench: A Benchmark Tool for VectorDB
 
+[![version](https://img.shields.io/pypi/v/vectordb-bench.svg?color=blue)](https://pypi.org/project/vectordb-bench/)
+[![Downloads](https://pepy.tech/badge/vectordb-bench)](https://pepy.tech/project/vectordb-bench)
+
+**Leaderboard:** https://zilliz.com/benchmark
 ## Quick Start
 ### Prerequirement
 ``` shell
@@ -34,10 +38,28 @@ VectorDBBench is not just an offering of benchmark results for mainstream vector
 Understanding the importance of user experience, we provide an intuitive visual interface. This not only empowers users to initiate benchmarks at ease, but also to view comparative result reports, thereby reproducing benchmark results effortlessly.
 To add more relevance and practicality, we provide cost-effectiveness reports particularly for cloud services. This allows for a more realistic and applicable benchmarking process.
 
-Closely mimicking real-world production environments, we've set up diverse testing scenarios including insertion, searching, and filtered searching. To provide you with credible and reliable data, we've included public datasets from actual production scenarios, such as SIFT, GIST, Cohere, and more. It's fascinating to discover how a relatively unknown open-source database might excel in certain circumstances!
+Closely mimicking real-world production environments, we've set up diverse testing scenarios including insertion, searching, and filtered searching. To provide you with credible and reliable data, we've included public datasets from actual production scenarios, such as [SIFT](http://corpus-texmex.irisa.fr/), [GIST](http://corpus-texmex.irisa.fr/), [Cohere](https://huggingface.co/datasets/Cohere/wikipedia-22-12/tree/main/en), and more. It's fascinating to discover how a relatively unknown open-source database might excel in certain circumstances!
 
 Prepare to delve into the world of VectorDBBench, and let it guide you in uncovering your perfect vector database match.  
 
+## Leaderboard
+### Introduction
+To facilitate the presentation of test results and provide a comprehensive performance analysis report, we offer a [leaderboard page](https://zilliz.com/benchmark). It allows us to choose from QPS, QP$, and latency metrics, and provides a comprehensive assessment of a system's performance based on the test results of various cases and a set of scoring mechanisms (to be introduced later). On this leaderboard, we can select the systems and models to be compared, and filter out cases we do not want to consider. Comprehensive scores are always ranked from best to worst, and the specific test results of each query will be presented in the list below.
+
+### Scoring Rules
+
+1. For each case, select a base value and score each system based on relative values. 
+    - For QPS and QP$, we use the highest value as the reference, denoted as `base_QPS` or `base_QP$`, and the score of each system is `(QPS/base_QPS) * 100` or `(QP$/base_QP$) * 100`. 
+    - For Latency, we use the lowest value as the reference, that is, `base_Latency`, and the score of each system is `(Latency + 10ms)/(base_Latency + 10ms)`. 
+
+    We want to give equal weight to different cases, and not let a case with high absolute result values become the sole reason for the overall scoring. Therefore, when scoring different systems in each case, we need to use relative values. 
+
+    Also, for Latency, we add 10ms to the numerator and denominator to ensure that if every system performs particularly well in a case, its advantage will not be infinitely magnified when latency tends to 0.
+
+2. For systems that fail or timeout in a particular case, we will give them a score based on a value worse than the worst result by a factor of two. For example, in QPS or QP$, it would be half the lowest value. For Latency, it would be twice the maximum value.
+
+3. For each system, we will take the geometric mean of its scores in all cases as its comprehensive score for a particular metric.
+
 ## Build on your own
 ### Install requirements
 ``` shell
@@ -66,12 +88,14 @@ $ ruff check vectordb_bench --fix
 
 ## How does it work?
 ### Result Page
-![image](https://github.com/liliu-z/VectorDBBench/assets/105927039/a8418fb6-0822-4f04-a04d-ab9143815b3e)
+![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/7f5cdae7-f9f2-4a81-b2e0-e5c6268cd970)
 This is the main page of VectorDBBench, which displays the standard benchmark results we provide. Additionally, results of all tests performed by users themselves will also be shown here. We also offer the ability to select and compare results from multiple tests simultaneously.
 
-The standard benchmark results displayed here include all 12 cases that we currently support for all our clients (Milvus, Zilliz Cloud, Elastic Search, Qdrant Cloud, and Weaviate Cloud). However, as some systems may not be able to complete all the tests successfully due to issues like Out of Memory (OOM) or timeouts, not all clients are included in every case.
+The standard benchmark results displayed here include all 9 cases that we currently support for all our clients (Milvus, Zilliz Cloud, Elastic Search, Qdrant Cloud, and Weaviate Cloud). However, as some systems may not be able to complete all the tests successfully due to issues like Out of Memory (OOM) or timeouts, not all clients are included in every case.
+
+All standard benchmark results are generated by a client running on an 8 core, 32 GB host, which is located in the same region as the server being tested. The client host is equipped with an `Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz` processor. Also all the servers for the open-source systems tested in our benchmarks run on hosts with the same type of processor.
 ### Run Test Page
-![image](https://github.com/liliu-z/VectorDBBench/assets/105927039/364e2462-107e-4ce1-aabc-ff2b217ae9b7)
+![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/a789099a-3707-4214-8052-b73463b8f2c6)
 This is the page to run a test:
 1. Initially, you select the systems to be tested - multiple selections are allowed. Once selected, corresponding forms will pop up to gather necessary information for using the chosen databases. The db_label is used to differentiate different instances of the same system. We recommend filling in the host size or instance type here (as we do in our standard results).
 2. The next step is to select the test cases you want to perform. You can select multiple cases at once, and a form to collect corresponding parameters will appear.
@@ -80,11 +104,11 @@ Now we can only run one task at the same time.
 
 ## Module
 ### Code Structure
-![image](https://github.com/liliu-z/VectorDBBench/assets/105927039/8d65c8b4-b9a3-4405-9db8-d27bf1ffda4b)
+![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/8c06512e-5419-4381-b084-9c93aed59639)
 ### Client
 Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant, and Weaviate. Stay tuned for more options, as we are consistently working on extending our reach to other systems.
 ### Benchmark Cases
-We've developed an array of 12 comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into three main types:
+We've developed an array of 9 comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into three main types:
 #### Capacity Case
 - **Large Dim:** Tests the database's loading capacity by inserting large-dimension vectors (GIST 100K vectors, 960 dimensions) until fully loaded. The final number of inserted vectors is reported.
 - **Small Dim:** Similar to the Large Dim case but uses small-dimension vectors (SIFT 100K vectors, 128 dimensions).
@@ -92,30 +116,24 @@ We've developed an array of 12 comprehensive benchmark cases to test vector data
 - **XLarge Dataset:** Measures search performance with a massive dataset (LAION 100M vectors, 768 dimensions) at varying parallel levels. The results include index building time, recall, latency, and maximum QPS.
 - **Large Dataset:** Similar to the XLarge Dataset case, but uses a slightly smaller dataset (Cohere 10M vectors, 768 dimensions).
 - **Medium Dataset:** A case using a medium dataset (Cohere 1M vectors, 768 dimensions).
-- **Small Dataset:** This case uses a small dataset (Cohere 100K vectors, 768 dimensions).
 #### Filtering Search Performance Case
 - **Large Dataset, Low Filtering Rate:** Evaluates search performance with a large dataset (Cohere 10M vectors, 768 dimensions) under a low filtering rate (1% vectors) at different parallel levels.
 - **Medium Dataset, Low Filtering Rate:** This case uses a medium dataset (Cohere 1M vectors, 768 dimensions) with a similar low filtering rate.
-- **Small Dataset, Low Filtering Rate:** This case uses a small dataset (Cohere 100K vectors, 768 dimensions) with a low filtering rate.
 - **Large Dataset, High Filtering Rate:** It tests with a large dataset (Cohere 10M vectors, 768 dimensions) but under a high filtering rate (99% vectors).
 - **Medium Dataset, High Filtering Rate:** This case uses a medium dataset (Cohere 1M vectors, 768 dimensions) with a high filtering rate.
-- **Small Dataset, High Filtering Rate:** Finally, this case uses a small dataset (Cohere 100K vectors, 768 dimensions) under a high filtering rate.
 For a quick reference, here is a table summarizing the key aspects of each case:
 
-Case No. | Case Type | Dataset Size | Dataset Type | Filtering Rate | Results |
-|----------|-----------|--------------|--------------|----------------|---------|
-1 | Capacity Case | Large Dim | GIST 100K vectors, 960 dimensions | N/A | Number of inserted vectors |
-2 | Capacity Case | Small Dim | SIFT 100K vectors, 128 dimensions | N/A | Number of inserted vectors |
-3 | Search Performance Case | XLarge Dataset | LAION 100M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS |
-4 | Search Performance Case | Large Dataset | Cohere 10M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS |
-5 | Search Performance Case | Medium Dataset | Cohere 1M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS |
-6 | Search Performance Case | Small Dataset | Cohere 100K vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS |
-7 | Filtering Search Performance Case | Large Dataset, Low Filtering Rate | Cohere 10M vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS |
-8 | Filtering Search Performance Case | Medium Dataset, Low Filtering Rate | Cohere 1M vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS |
-9 | Filtering Search Performance Case | Small Dataset, Low Filtering Rate | Cohere 100K vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS |
-10 | Filtering Search Performance Case | Large Dataset, High Filtering Rate | Cohere 10M vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS |
-11 | Filtering Search Performance Case | Medium Dataset, High Filtering Rate | Cohere 1M vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS |
-12 | Filtering Search Performance Case | Small Dataset, High Filtering Rate | Cohere 100K vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS |
+Case No. | Case Type | Dataset Size  | Filtering Rate | Results |
+|----------|-----------|--------------|----------------|---------|
+1 | Capacity Case | GIST 100K vectors, 960 dimensions | N/A | Number of inserted vectors |
+2 | Capacity Case | SIFT 100K vectors, 128 dimensions | N/A | Number of inserted vectors |
+3 | Search Performance Case | LAION 100M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS |
+4 | Search Performance Case | Cohere 10M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS |
+5 | Search Performance Case | Cohere 1M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS |
+6 | Filtering Search Performance Case | Cohere 10M vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS |
+7 | Filtering Search Performance Case | Cohere 1M vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS |
+8 | Filtering Search Performance Case | Cohere 10M vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS |
+9 | Filtering Search Performance Case | Cohere 1M vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS |
 
 Each case provides an in-depth examination of a vector database's abilities, providing you a comprehensive view of the database's performance.
 
@@ -136,7 +154,7 @@ VectorDBBench aims to provide a more comprehensive, multi-faceted testing enviro
 
 1. Navigate to the vectordb_bench/backend/clients directory.
 2. Create a new folder for your client, for example, "new_client".
-3. Inside the "new_client" folder, create two files: new_client.py and config.py. 
+3. Inside the "new_client" folder, create two files: new_client.py and config.py.
 
 **Step 2: Implement new_client.py and config.py**
 
@@ -201,7 +219,66 @@ For the system under test, we use the default server-side configuration to maint
 For the Client, we welcome any parameter tuning to obtain better results.
 ### Incomplete Results
 Many databases may not be able to complete all test cases due to issues such as Out of Memory (OOM), crashes, or timeouts. In these scenarios, we will clearly state these occurrences in the test results.
-### Unpublishable Results
-Although we are trying to support as many clients as possible for benchmarking, due to the restrictions imposed by the [Dewitt Clause](https://cube.dev/blog/dewitt-clause-or-can-you-benchmark-a-database), we're unable to publish all benchmark results. This means that users may not be able to fully compare performance data for certain databases on our platform, despite the support we have integrated for these systems. 
 ### Mistake Or Misrepresentation 
 We strive for accuracy in learning and supporting various vector databases, yet there might be oversights or misapplications. For any such occurrences, feel free to [raise an issue](https://github.com/zilliztech/VectorDBBench/issues/new) or make amendments on our GitHub page.
+## Timeout
+In our pursuit to ensure that our benchmark reflects the reality of a production environment while guaranteeing the practicality of the system, we have implemented a timeout plan based on our experiences for various tests.
+
+**1. Capacity Case:**
+- For Capacity Case, we have assigned an overall timeout.
+
+**2. Other Cases:**
+
+For other cases, we have set two timeouts:
+
+- **Data Loading Timeout:** This timeout is designed to filter out systems that are too slow in inserting data, thus ensuring that we are only considering systems that is able to cope with the demands of a real-world production environment within a reasonable time frame.
+
+- **Optimization Preparation Timeout**: This timeout is established to avoid excessive optimization strategies that might work for benchmarks but fail to deliver in real production environments. By doing this, we ensure that the systems we consider are not only suitable for testing environments but also applicable and efficient in production scenarios.
+
+This multi-tiered timeout approach allows our benchmark to be more representative of actual production environments and assists us in identifying systems that can truly perform in real-world scenarios.
+<table>
+    <tr>
+        <th>Case</th>
+        <th>Data Size</th>
+        <th>Timeout Type</th>
+        <th>Value</th>
+    </tr>
+    <tr>
+        <td>Capacity Case</td>
+        <td>N/A</td>
+        <td>Loading timeout</td>
+        <td>24 hours</td>
+    </tr>
+    <tr>
+        <td rowspan="2">Other Cases</td>
+        <td rowspan="2">1M vectors, 768 dimensions</td>
+        <td>Loading timeout</td>
+        <td>2.5 hours</td>
+    </tr>
+    <tr>
+        <td>Optimization timeout</td>
+        <td>15 mins</td>
+    </tr>
+    <tr>
+        <td rowspan="2">Other Cases</td>
+        <td rowspan="2">10M vectors, 768 dimensions</td>
+        <td>Loading timeout</td>
+        <td>25 hours</td>
+    </tr>
+    <tr>
+        <td>Optimization timeout</td>
+        <td>2.5 hours</td>
+    </tr>
+    <tr>
+        <td rowspan="2">Other Cases</td>
+        <td rowspan="2">100M vectors, 768 dimensions</td>
+        <td>Loading timeout</td>
+        <td>250 hours</td>
+    </tr>
+    <tr>
+        <td>Optimization timeout</td>
+        <td>25 hours</td>
+    </tr>
+</table>
+
+**Note:** Some datapoints in the standard benchmark results that voilate this timeout will be kept for now for reference. We will remove them in the future.

{vectordb-bench-0.0.1 → vectordb-bench-0.0.3}/README.md

@@ -1,5 +1,9 @@
 # VectorDBBench: A Benchmark Tool for VectorDB
 
+[![version](https://img.shields.io/pypi/v/vectordb-bench.svg?color=blue)](https://pypi.org/project/vectordb-bench/)
+[![Downloads](https://pepy.tech/badge/vectordb-bench)](https://pepy.tech/project/vectordb-bench)
+
+**Leaderboard:** https://zilliz.com/benchmark
 ## Quick Start
 ### Prerequirement
 ``` shell
@@ -20,10 +24,28 @@ VectorDBBench is not just an offering of benchmark results for mainstream vector
 Understanding the importance of user experience, we provide an intuitive visual interface. This not only empowers users to initiate benchmarks at ease, but also to view comparative result reports, thereby reproducing benchmark results effortlessly.
 To add more relevance and practicality, we provide cost-effectiveness reports particularly for cloud services. This allows for a more realistic and applicable benchmarking process.
 
-Closely mimicking real-world production environments, we've set up diverse testing scenarios including insertion, searching, and filtered searching. To provide you with credible and reliable data, we've included public datasets from actual production scenarios, such as SIFT, GIST, Cohere, and more. It's fascinating to discover how a relatively unknown open-source database might excel in certain circumstances!
+Closely mimicking real-world production environments, we've set up diverse testing scenarios including insertion, searching, and filtered searching. To provide you with credible and reliable data, we've included public datasets from actual production scenarios, such as [SIFT](http://corpus-texmex.irisa.fr/), [GIST](http://corpus-texmex.irisa.fr/), [Cohere](https://huggingface.co/datasets/Cohere/wikipedia-22-12/tree/main/en), and more. It's fascinating to discover how a relatively unknown open-source database might excel in certain circumstances!
 
 Prepare to delve into the world of VectorDBBench, and let it guide you in uncovering your perfect vector database match.  
 
+## Leaderboard
+### Introduction
+To facilitate the presentation of test results and provide a comprehensive performance analysis report, we offer a [leaderboard page](https://zilliz.com/benchmark). It allows us to choose from QPS, QP$, and latency metrics, and provides a comprehensive assessment of a system's performance based on the test results of various cases and a set of scoring mechanisms (to be introduced later). On this leaderboard, we can select the systems and models to be compared, and filter out cases we do not want to consider. Comprehensive scores are always ranked from best to worst, and the specific test results of each query will be presented in the list below.
+
+### Scoring Rules
+
+1. For each case, select a base value and score each system based on relative values. 
+    - For QPS and QP$, we use the highest value as the reference, denoted as `base_QPS` or `base_QP$`, and the score of each system is `(QPS/base_QPS) * 100` or `(QP$/base_QP$) * 100`. 
+    - For Latency, we use the lowest value as the reference, that is, `base_Latency`, and the score of each system is `(Latency + 10ms)/(base_Latency + 10ms)`. 
+
+    We want to give equal weight to different cases, and not let a case with high absolute result values become the sole reason for the overall scoring. Therefore, when scoring different systems in each case, we need to use relative values. 
+
+    Also, for Latency, we add 10ms to the numerator and denominator to ensure that if every system performs particularly well in a case, its advantage will not be infinitely magnified when latency tends to 0.
+
+2. For systems that fail or timeout in a particular case, we will give them a score based on a value worse than the worst result by a factor of two. For example, in QPS or QP$, it would be half the lowest value. For Latency, it would be twice the maximum value.
+
+3. For each system, we will take the geometric mean of its scores in all cases as its comprehensive score for a particular metric.
+
 ## Build on your own
 ### Install requirements
 ``` shell
@@ -52,12 +74,14 @@ $ ruff check vectordb_bench --fix
 
 ## How does it work?
 ### Result Page
-![image](https://github.com/liliu-z/VectorDBBench/assets/105927039/a8418fb6-0822-4f04-a04d-ab9143815b3e)
+![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/7f5cdae7-f9f2-4a81-b2e0-e5c6268cd970)
 This is the main page of VectorDBBench, which displays the standard benchmark results we provide. Additionally, results of all tests performed by users themselves will also be shown here. We also offer the ability to select and compare results from multiple tests simultaneously.
 
-The standard benchmark results displayed here include all 12 cases that we currently support for all our clients (Milvus, Zilliz Cloud, Elastic Search, Qdrant Cloud, and Weaviate Cloud). However, as some systems may not be able to complete all the tests successfully due to issues like Out of Memory (OOM) or timeouts, not all clients are included in every case.
+The standard benchmark results displayed here include all 9 cases that we currently support for all our clients (Milvus, Zilliz Cloud, Elastic Search, Qdrant Cloud, and Weaviate Cloud). However, as some systems may not be able to complete all the tests successfully due to issues like Out of Memory (OOM) or timeouts, not all clients are included in every case.
+
+All standard benchmark results are generated by a client running on an 8 core, 32 GB host, which is located in the same region as the server being tested. The client host is equipped with an `Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz` processor. Also all the servers for the open-source systems tested in our benchmarks run on hosts with the same type of processor.
 ### Run Test Page
-![image](https://github.com/liliu-z/VectorDBBench/assets/105927039/364e2462-107e-4ce1-aabc-ff2b217ae9b7)
+![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/a789099a-3707-4214-8052-b73463b8f2c6)
 This is the page to run a test:
 1. Initially, you select the systems to be tested - multiple selections are allowed. Once selected, corresponding forms will pop up to gather necessary information for using the chosen databases. The db_label is used to differentiate different instances of the same system. We recommend filling in the host size or instance type here (as we do in our standard results).
 2. The next step is to select the test cases you want to perform. You can select multiple cases at once, and a form to collect corresponding parameters will appear.
@@ -66,11 +90,11 @@ Now we can only run one task at the same time.
 
 ## Module
 ### Code Structure
-![image](https://github.com/liliu-z/VectorDBBench/assets/105927039/8d65c8b4-b9a3-4405-9db8-d27bf1ffda4b)
+![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/8c06512e-5419-4381-b084-9c93aed59639)
 ### Client
 Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant, and Weaviate. Stay tuned for more options, as we are consistently working on extending our reach to other systems.
 ### Benchmark Cases
-We've developed an array of 12 comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into three main types:
+We've developed an array of 9 comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into three main types:
 #### Capacity Case
 - **Large Dim:** Tests the database's loading capacity by inserting large-dimension vectors (GIST 100K vectors, 960 dimensions) until fully loaded. The final number of inserted vectors is reported.
 - **Small Dim:** Similar to the Large Dim case but uses small-dimension vectors (SIFT 100K vectors, 128 dimensions).
@@ -78,30 +102,24 @@ We've developed an array of 12 comprehensive benchmark cases to test vector data
 - **XLarge Dataset:** Measures search performance with a massive dataset (LAION 100M vectors, 768 dimensions) at varying parallel levels. The results include index building time, recall, latency, and maximum QPS.
 - **Large Dataset:** Similar to the XLarge Dataset case, but uses a slightly smaller dataset (Cohere 10M vectors, 768 dimensions).
 - **Medium Dataset:** A case using a medium dataset (Cohere 1M vectors, 768 dimensions).
-- **Small Dataset:** This case uses a small dataset (Cohere 100K vectors, 768 dimensions).
 #### Filtering Search Performance Case
 - **Large Dataset, Low Filtering Rate:** Evaluates search performance with a large dataset (Cohere 10M vectors, 768 dimensions) under a low filtering rate (1% vectors) at different parallel levels.
 - **Medium Dataset, Low Filtering Rate:** This case uses a medium dataset (Cohere 1M vectors, 768 dimensions) with a similar low filtering rate.
-- **Small Dataset, Low Filtering Rate:** This case uses a small dataset (Cohere 100K vectors, 768 dimensions) with a low filtering rate.
 - **Large Dataset, High Filtering Rate:** It tests with a large dataset (Cohere 10M vectors, 768 dimensions) but under a high filtering rate (99% vectors).
 - **Medium Dataset, High Filtering Rate:** This case uses a medium dataset (Cohere 1M vectors, 768 dimensions) with a high filtering rate.
-- **Small Dataset, High Filtering Rate:** Finally, this case uses a small dataset (Cohere 100K vectors, 768 dimensions) under a high filtering rate.
 For a quick reference, here is a table summarizing the key aspects of each case:
 
-Case No. | Case Type | Dataset Size | Dataset Type | Filtering Rate | Results |
-|----------|-----------|--------------|--------------|----------------|---------|
-1 | Capacity Case | Large Dim | GIST 100K vectors, 960 dimensions | N/A | Number of inserted vectors |
-2 | Capacity Case | Small Dim | SIFT 100K vectors, 128 dimensions | N/A | Number of inserted vectors |
-3 | Search Performance Case | XLarge Dataset | LAION 100M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS |
-4 | Search Performance Case | Large Dataset | Cohere 10M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS |
-5 | Search Performance Case | Medium Dataset | Cohere 1M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS |
-6 | Search Performance Case | Small Dataset | Cohere 100K vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS |
-7 | Filtering Search Performance Case | Large Dataset, Low Filtering Rate | Cohere 10M vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS |
-8 | Filtering Search Performance Case | Medium Dataset, Low Filtering Rate | Cohere 1M vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS |
-9 | Filtering Search Performance Case | Small Dataset, Low Filtering Rate | Cohere 100K vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS |
-10 | Filtering Search Performance Case | Large Dataset, High Filtering Rate | Cohere 10M vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS |
-11 | Filtering Search Performance Case | Medium Dataset, High Filtering Rate | Cohere 1M vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS |
-12 | Filtering Search Performance Case | Small Dataset, High Filtering Rate | Cohere 100K vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS |
+Case No. | Case Type | Dataset Size  | Filtering Rate | Results |
+|----------|-----------|--------------|----------------|---------|
+1 | Capacity Case | GIST 100K vectors, 960 dimensions | N/A | Number of inserted vectors |
+2 | Capacity Case | SIFT 100K vectors, 128 dimensions | N/A | Number of inserted vectors |
+3 | Search Performance Case | LAION 100M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS |
+4 | Search Performance Case | Cohere 10M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS |
+5 | Search Performance Case | Cohere 1M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS |
+6 | Filtering Search Performance Case | Cohere 10M vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS |
+7 | Filtering Search Performance Case | Cohere 1M vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS |
+8 | Filtering Search Performance Case | Cohere 10M vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS |
+9 | Filtering Search Performance Case | Cohere 1M vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS |
 
 Each case provides an in-depth examination of a vector database's abilities, providing you a comprehensive view of the database's performance.
 
@@ -122,7 +140,7 @@ VectorDBBench aims to provide a more comprehensive, multi-faceted testing enviro
 
 1. Navigate to the vectordb_bench/backend/clients directory.
 2. Create a new folder for your client, for example, "new_client".
-3. Inside the "new_client" folder, create two files: new_client.py and config.py. 
+3. Inside the "new_client" folder, create two files: new_client.py and config.py.
 
 **Step 2: Implement new_client.py and config.py**
 
@@ -187,7 +205,66 @@ For the system under test, we use the default server-side configuration to maint
 For the Client, we welcome any parameter tuning to obtain better results.
 ### Incomplete Results
 Many databases may not be able to complete all test cases due to issues such as Out of Memory (OOM), crashes, or timeouts. In these scenarios, we will clearly state these occurrences in the test results.
-### Unpublishable Results
-Although we are trying to support as many clients as possible for benchmarking, due to the restrictions imposed by the [Dewitt Clause](https://cube.dev/blog/dewitt-clause-or-can-you-benchmark-a-database), we're unable to publish all benchmark results. This means that users may not be able to fully compare performance data for certain databases on our platform, despite the support we have integrated for these systems. 
 ### Mistake Or Misrepresentation 
 We strive for accuracy in learning and supporting various vector databases, yet there might be oversights or misapplications. For any such occurrences, feel free to [raise an issue](https://github.com/zilliztech/VectorDBBench/issues/new) or make amendments on our GitHub page.
+## Timeout
+In our pursuit to ensure that our benchmark reflects the reality of a production environment while guaranteeing the practicality of the system, we have implemented a timeout plan based on our experiences for various tests.
+
+**1. Capacity Case:**
+- For Capacity Case, we have assigned an overall timeout.
+
+**2. Other Cases:**
+
+For other cases, we have set two timeouts:
+
+- **Data Loading Timeout:** This timeout is designed to filter out systems that are too slow in inserting data, thus ensuring that we are only considering systems that is able to cope with the demands of a real-world production environment within a reasonable time frame.
+
+- **Optimization Preparation Timeout**: This timeout is established to avoid excessive optimization strategies that might work for benchmarks but fail to deliver in real production environments. By doing this, we ensure that the systems we consider are not only suitable for testing environments but also applicable and efficient in production scenarios.
+
+This multi-tiered timeout approach allows our benchmark to be more representative of actual production environments and assists us in identifying systems that can truly perform in real-world scenarios.
+<table>
+    <tr>
+        <th>Case</th>
+        <th>Data Size</th>
+        <th>Timeout Type</th>
+        <th>Value</th>
+    </tr>
+    <tr>
+        <td>Capacity Case</td>
+        <td>N/A</td>
+        <td>Loading timeout</td>
+        <td>24 hours</td>
+    </tr>
+    <tr>
+        <td rowspan="2">Other Cases</td>
+        <td rowspan="2">1M vectors, 768 dimensions</td>
+        <td>Loading timeout</td>
+        <td>2.5 hours</td>
+    </tr>
+    <tr>
+        <td>Optimization timeout</td>
+        <td>15 mins</td>
+    </tr>
+    <tr>
+        <td rowspan="2">Other Cases</td>
+        <td rowspan="2">10M vectors, 768 dimensions</td>
+        <td>Loading timeout</td>
+        <td>25 hours</td>
+    </tr>
+    <tr>
+        <td>Optimization timeout</td>
+        <td>2.5 hours</td>
+    </tr>
+    <tr>
+        <td rowspan="2">Other Cases</td>
+        <td rowspan="2">100M vectors, 768 dimensions</td>
+        <td>Loading timeout</td>
+        <td>250 hours</td>
+    </tr>
+    <tr>
+        <td>Optimization timeout</td>
+        <td>25 hours</td>
+    </tr>
+</table>
+
+**Note:** Some datapoints in the standard benchmark results that voilate this timeout will be kept for now for reference. We will remove them in the future.

{vectordb-bench-0.0.1 → vectordb-bench-0.0.3}/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=67.0", "wheel"]
+requires = ["setuptools>=67.0", "wheel", "setuptools_scm[toml]>=6.2"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools.package-data]
@@ -37,8 +37,11 @@ dependencies = [
     "scikit-learn",
     "s3fs",
     "psutil",
+    "polars",
+    "pgvector",
+    "sqlalchemy"
 ]
-version = "0.0.1"
+dynamic = ["version"]
 
 [project.optional-dependencies]
 test = [
@@ -51,3 +54,5 @@ test = [
 
 [project.scripts]
 init_bench = "vectordb_bench.__main__:main"
+
+[tool.setuptools_scm]

vectordb-bench-0.0.3/tests/conftest.py

@@ -0,0 +1,4 @@
+import sys
+
+from os.path import dirname, abspath
+sys.path.append(dirname(dirname(abspath(__file__))))

vectordb-bench-0.0.3/tests/pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+
+filterwarnings = 
+    ignore::UserWarning

vectordb-bench-0.0.3/tests/test_dataset.py

@@ -0,0 +1,36 @@
+from vectordb_bench.backend.dataset import Dataset
+import logging
+import pytest
+from pydantic import ValidationError
+
+
+log = logging.getLogger("vectordb_bench")
+
+class TestDataSet:
+    def test_iter_dataset(self):
+        for ds in Dataset:
+            log.info(ds)
+
+    def test_cohere(self):
+        cohere = Dataset.COHERE.get(100_000)
+        log.info(cohere)
+        assert cohere.name == "Cohere"
+        assert cohere.size == 100_000
+        assert cohere.label == "SMALL"
+        assert cohere.dim == 768
+
+    def test_cohere_error(self):
+        with pytest.raises(ValidationError):
+            Dataset.COHERE.get(9999)
+
+    def test_init_cohere(self):
+        coheres = [Dataset.COHERE.manager(i) for i in [100_000, 1_000_000, 10_000_000]]
+        for t in coheres:
+            t._validate_local_file()
+
+    def test_iter_cohere(self):
+        cohere_10m = Dataset.COHERE.manager(10_000_000)
+        cohere_10m.prepare(False)
+        for i in cohere_10m:
+            log.debug(i.head(1))
+

{vectordb-bench-0.0.1 → vectordb-bench-0.0.3}/tests/test_models.py

@@ -24,7 +24,7 @@ class TestModels:
                 db=DB.Milvus,
                 db_config=DB.Milvus.config(),
                 db_case_config=DB.Milvus.case_config_cls(index=IndexType.Flat)(),
-                case_config=CaseConfig(case_id=CaseType.PerformanceLZero),
+                case_config=CaseConfig(case_id=CaseType.Performance10M),
             ),
             metrics=Metric(),
         )
@@ -65,6 +65,6 @@ class TestModels:
 
     def test_test_result_display(self):
         result_dir = config.RESULTS_LOCAL_DIR
-        for json_file in result_dir.glob("*.json"):
+        for json_file in result_dir.glob("result*.json"):
             res = TestResult.read_file(json_file)
             res.display()