vectordb-bench 0.0.21__py3-none-any.whl → 0.0.22__py3-none-any.whl

vectordb_bench/backend/clients/__init__.py

@@ -40,6 +40,7 @@ class DB(Enum):
     AliyunElasticsearch = "AliyunElasticsearch"
     Test = "test"
     AliyunOpenSearch = "AliyunOpenSearch"
+    MongoDB = "MongoDB"
 
     @property
     def init_cls(self) -> type[VectorDB]:  # noqa: PLR0911, PLR0912, C901
@@ -129,6 +130,11 @@ class DB(Enum):
 
             return AliyunOpenSearch
 
+        if self == DB.MongoDB:
+            from .mongodb.mongodb import MongoDB
+
+            return MongoDB
+
         if self == DB.Test:
             from .test.test import Test
 
@@ -225,6 +231,11 @@ class DB(Enum):
 
             return AliyunOpenSearchConfig
 
+        if self == DB.MongoDB:
+            from .mongodb.config import MongoDBConfig
+
+            return MongoDBConfig
+
         if self == DB.Test:
             from .test.config import TestConfig
 
@@ -302,6 +313,11 @@ class DB(Enum):
 
             return AliyunOpenSearchIndexConfig
 
+        if self == DB.MongoDB:
+            from .mongodb.config import MongoDBIndexConfig
+
+            return MongoDBIndexConfig
+
         # DB.Pinecone, DB.Chroma, DB.Redis
         return EmptyDBCaseConfig
 

vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py

@@ -12,6 +12,7 @@ log = logging.getLogger(__name__)
 
 WAITING_FOR_REFRESH_SEC = 30
 WAITING_FOR_FORCE_MERGE_SEC = 30
+SECONDS_WAITING_FOR_REPLICAS_TO_BE_ENABLED_SEC = 30
 
 
 class AWSOpenSearch(VectorDB):
@@ -52,10 +53,27 @@ class AWSOpenSearch(VectorDB):
         return AWSOpenSearchIndexConfig
 
     def _create_index(self, client: OpenSearch):
+        cluster_settings_body = {
+            "persistent": {
+                "knn.algo_param.index_thread_qty": self.case_config.index_thread_qty,
+                "knn.memory.circuit_breaker.limit": self.case_config.cb_threshold,
+            }
+        }
+        client.cluster.put_settings(cluster_settings_body)
         settings = {
             "index": {
                 "knn": True,
+                "number_of_shards": self.case_config.number_of_shards,
+                "number_of_replicas": 0,
+                "translog.flush_threshold_size": self.case_config.flush_threshold_size,
+                # Setting trans log threshold to 5GB
+                **(
+                    {"knn.algo_param.ef_search": self.case_config.ef_search}
+                    if self.case_config.engine == AWSOS_Engine.nmslib
+                    else {}
+                ),
             },
+            "refresh_interval": self.case_config.refresh_interval,
         }
         mappings = {
             "properties": {
@@ -145,9 +163,9 @@ class AWSOpenSearch(VectorDB):
                 docvalue_fields=[self.id_col_name],
                 stored_fields="_none_",
             )
-            log.info(f"Search took: {resp['took']}")
-            log.info(f"Search shards: {resp['_shards']}")
-            log.info(f"Search hits total: {resp['hits']['total']}")
+            log.debug(f"Search took: {resp['took']}")
+            log.debug(f"Search shards: {resp['_shards']}")
+            log.debug(f"Search hits total: {resp['hits']['total']}")
             return [int(h["fields"][self.id_col_name][0]) for h in resp["hits"]["hits"]]
         except Exception as e:
             log.warning(f"Failed to search: {self.index_name} error: {e!s}")
@@ -157,12 +175,37 @@ class AWSOpenSearch(VectorDB):
         """optimize will be called between insertion and search in performance cases."""
         # Call refresh first to ensure that all segments are created
         self._refresh_index()
-        self._do_force_merge()
+        if self.case_config.force_merge_enabled:
+            self._do_force_merge()
+            self._refresh_index()
+        self._update_replicas()
         # Call refresh again to ensure that the index is ready after force merge.
         self._refresh_index()
         # ensure that all graphs are loaded in memory and ready for search
         self._load_graphs_to_memory()
 
+    def _update_replicas(self):
+        index_settings = self.client.indices.get_settings(index=self.index_name)
+        current_number_of_replicas = int(index_settings[self.index_name]["settings"]["index"]["number_of_replicas"])
+        log.info(
+            f"Current Number of replicas are {current_number_of_replicas}"
+            f" and changing the replicas to {self.case_config.number_of_replicas}"
+        )
+        settings_body = {"index": {"number_of_replicas": self.case_config.number_of_replicas}}
+        self.client.indices.put_settings(index=self.index_name, body=settings_body)
+        self._wait_till_green()
+
+    def _wait_till_green(self):
+        log.info("Wait for index to become green..")
+        while True:
+            res = self.client.cat.indices(index=self.index_name, h="health", format="json")
+            health = res[0]["health"]
+            if health != "green":
+                break
+            log.info(f"The index {self.index_name} has health : {health} and is not green. Retrying")
+            time.sleep(SECONDS_WAITING_FOR_REPLICAS_TO_BE_ENABLED_SEC)
+        log.info(f"Index {self.index_name} is green..")
+
     def _refresh_index(self):
         log.debug(f"Starting refresh for index {self.index_name}")
         while True:
@@ -179,6 +222,12 @@ class AWSOpenSearch(VectorDB):
         log.debug(f"Completed refresh for index {self.index_name}")
 
     def _do_force_merge(self):
+        log.info(f"Updating the Index thread qty to {self.case_config.index_thread_qty_during_force_merge}.")
+
+        cluster_settings_body = {
+            "persistent": {"knn.algo_param.index_thread_qty": self.case_config.index_thread_qty_during_force_merge}
+        }
+        self.client.cluster.put_settings(cluster_settings_body)
         log.debug(f"Starting force merge for index {self.index_name}")
         force_merge_endpoint = f"/{self.index_name}/_forcemerge?max_num_segments=1&wait_for_completion=false"
         force_merge_task_id = self.client.transport.perform_request("POST", force_merge_endpoint)["task"]

vectordb_bench/backend/clients/aws_opensearch/cli.py

@@ -18,6 +18,79 @@ class AWSOpenSearchTypedDict(TypedDict):
     port: Annotated[int, click.option("--port", type=int, default=443, help="Db Port")]
     user: Annotated[str, click.option("--user", type=str, default="admin", help="Db User")]
     password: Annotated[str, click.option("--password", type=str, help="Db password")]
+    number_of_shards: Annotated[
+        int,
+        click.option("--number-of-shards", type=int, help="Number of primary shards for the index", default=1),
+    ]
+    number_of_replicas: Annotated[
+        int,
+        click.option(
+            "--number-of-replicas", type=int, help="Number of replica copies for each primary shard", default=1
+        ),
+    ]
+    index_thread_qty: Annotated[
+        int,
+        click.option(
+            "--index-thread-qty",
+            type=int,
+            help="Thread count for native engine indexing",
+            default=4,
+        ),
+    ]
+
+    index_thread_qty_during_force_merge: Annotated[
+        int,
+        click.option(
+            "--index-thread-qty-during-force-merge",
+            type=int,
+            help="Thread count during force merge operations",
+            default=4,
+        ),
+    ]
+
+    number_of_indexing_clients: Annotated[
+        int,
+        click.option(
+            "--number-of-indexing-clients",
+            type=int,
+            help="Number of concurrent indexing clients",
+            default=1,
+        ),
+    ]
+
+    number_of_segments: Annotated[
+        int,
+        click.option("--number-of-segments", type=int, help="Target number of segments after merging", default=1),
+    ]
+
+    refresh_interval: Annotated[
+        int,
+        click.option(
+            "--refresh-interval", type=str, help="How often to make new data available for search", default="60s"
+        ),
+    ]
+
+    force_merge_enabled: Annotated[
+        int,
+        click.option("--force-merge-enabled", type=bool, help="Whether to perform force merge operation", default=True),
+    ]
+
+    flush_threshold_size: Annotated[
+        int,
+        click.option(
+            "--flush-threshold-size", type=str, help="Size threshold for flushing the transaction log", default="5120mb"
+        ),
+    ]
+
+    cb_threshold: Annotated[
+        int,
+        click.option(
+            "--cb-threshold",
+            type=str,
+            help="k-NN Memory circuit breaker threshold",
+            default="50%",
+        ),
+    ]
 
 
 class AWSOpenSearchHNSWTypedDict(CommonTypedDict, AWSOpenSearchTypedDict, HNSWFlavor2): ...
@@ -36,6 +109,17 @@ def AWSOpenSearch(**parameters: Unpack[AWSOpenSearchHNSWTypedDict]):
             user=parameters["user"],
             password=SecretStr(parameters["password"]),
         ),
-        db_case_config=AWSOpenSearchIndexConfig(),
+        db_case_config=AWSOpenSearchIndexConfig(
+            number_of_shards=parameters["number_of_shards"],
+            number_of_replicas=parameters["number_of_replicas"],
+            index_thread_qty=parameters["index_thread_qty"],
+            number_of_segments=parameters["number_of_segments"],
+            refresh_interval=parameters["refresh_interval"],
+            force_merge_enabled=parameters["force_merge_enabled"],
+            flush_threshold_size=parameters["flush_threshold_size"],
+            number_of_indexing_clients=parameters["number_of_indexing_clients"],
+            index_thread_qty_during_force_merge=parameters["index_thread_qty_during_force_merge"],
+            cb_threshold=parameters["cb_threshold"],
+        ),
         **parameters,
     )

vectordb_bench/backend/clients/aws_opensearch/config.py

@@ -39,6 +39,16 @@ class AWSOpenSearchIndexConfig(BaseModel, DBCaseConfig):
     efConstruction: int = 256
     efSearch: int = 256
     M: int = 16
+    index_thread_qty: int | None = 4
+    number_of_shards: int | None = 1
+    number_of_replicas: int | None = 0
+    number_of_segments: int | None = 1
+    refresh_interval: str | None = "60s"
+    force_merge_enabled: bool | None = True
+    flush_threshold_size: str | None = "5120mb"
+    number_of_indexing_clients: int | None = 1
+    index_thread_qty_during_force_merge: int
+    cb_threshold: str | None = "50%"
 
     def parse_metric(self) -> str:
         if self.metric_type == MetricType.IP:

vectordb_bench/backend/clients/mongodb/config.py

@@ -0,0 +1,53 @@
+from enum import Enum
+
+from pydantic import BaseModel, SecretStr
+
+from ..api import DBCaseConfig, DBConfig, IndexType, MetricType
+
+
+class QuantizationType(Enum):
+    NONE = "none"
+    BINARY = "binary"
+    SCALAR = "scalar"
+
+
+class MongoDBConfig(DBConfig, BaseModel):
+    connection_string: SecretStr = "mongodb+srv://<user>:<password>@<cluster_name>.heatl.mongodb.net"
+    database: str = "vdb_bench"
+
+    def to_dict(self) -> dict:
+        return {
+            "connection_string": self.connection_string.get_secret_value(),
+            "database": self.database,
+        }
+
+
+class MongoDBIndexConfig(BaseModel, DBCaseConfig):
+    index: IndexType = IndexType.HNSW  # MongoDB uses HNSW for vector search
+    metric_type: MetricType = MetricType.COSINE
+    num_candidates_ratio: int = 10  # Default numCandidates ratio for vector search
+    quantization: QuantizationType = QuantizationType.NONE  # Quantization type if applicable
+
+    def parse_metric(self) -> str:
+        if self.metric_type == MetricType.L2:
+            return "euclidean"
+        if self.metric_type == MetricType.IP:
+            return "dotProduct"
+        return "cosine"  # Default to cosine similarity
+
+    def index_param(self) -> dict:
+        return {
+            "type": "vectorSearch",
+            "fields": [
+                {
+                    "type": "vector",
+                    "similarity": self.parse_metric(),
+                    "numDimensions": None,  # Will be set in MongoDB class
+                    "path": "vector",  # Vector field name
+                    "quantization": self.quantization.value,
+                }
+            ],
+        }
+
+    def search_param(self) -> dict:
+        return {"num_candidates_ratio": self.num_candidates_ratio}

vectordb_bench/backend/clients/mongodb/mongodb.py

@@ -0,0 +1,200 @@
+import logging
+import time
+from contextlib import contextmanager
+
+from pymongo import MongoClient
+from pymongo.operations import SearchIndexModel
+
+from ..api import VectorDB
+from .config import MongoDBIndexConfig
+
+log = logging.getLogger(__name__)
+
+
+class MongoDBError(Exception):
+    """Custom exception class for MongoDB client errors."""
+
+
+class MongoDB(VectorDB):
+    def __init__(
+        self,
+        dim: int,
+        db_config: dict,
+        db_case_config: MongoDBIndexConfig,
+        collection_name: str = "vdb_bench_collection",
+        id_field: str = "id",
+        vector_field: str = "vector",
+        drop_old: bool = False,
+        **kwargs,
+    ):
+        self.dim = dim
+        self.db_config = db_config
+        self.case_config = db_case_config
+        self.collection_name = collection_name
+        self.id_field = id_field
+        self.vector_field = vector_field
+        self.drop_old = drop_old
+
+        # Update index dimensions
+        index_params = self.case_config.index_param()
+        log.info(f"index params: {index_params}")
+        index_params["fields"][0]["numDimensions"] = dim
+        self.index_params = index_params
+
+        # Initialize  - they'll also be set in init()
+        uri = self.db_config["connection_string"]
+        self.client = MongoClient(uri)
+        self.db = self.client[self.db_config["database"]]
+        self.collection = self.db[self.collection_name]
+        if self.drop_old and self.collection_name in self.db.list_collection_names():
+            log.info(f"MongoDB client dropping old collection: {self.collection_name}")
+            self.db.drop_collection(self.collection_name)
+        self.client = None
+        self.db = None
+        self.collection = None
+
+    @contextmanager
+    def init(self):
+        """Initialize MongoDB client and cleanup when done"""
+        try:
+            uri = self.db_config["connection_string"]
+            self.client = MongoClient(uri)
+            self.db = self.client[self.db_config["database"]]
+            self.collection = self.db[self.collection_name]
+
+            yield
+        finally:
+            if self.client is not None:
+                self.client.close()
+                self.client = None
+                self.db = None
+                self.collection = None
+
+    def _create_index(self) -> None:
+        """Create vector search index"""
+        index_name = "vector_index"
+        index_params = self.index_params
+        log.info(f"index params {index_params}")
+        # drop index if already exists
+        if self.collection.list_indexes():
+            all_indexes = self.collection.list_search_indexes()
+            if any(idx.get("name") == index_name for idx in all_indexes):
+                log.info(f"Drop index: {index_name}")
+                try:
+                    self.collection.drop_search_index(index_name)
+                    while True:
+                        indices = list(self.collection.list_search_indexes())
+                        indices = [idx for idx in indices if idx["name"] == index_name]
+                        log.debug(f"index status {indices}")
+                        if len(indices) == 0:
+                            break
+                        log.info(f"index deleting {indices}")
+                except Exception:
+                    log.exception(f"Error dropping index {index_name}")
+        try:
+            # Create vector search index
+            search_index = SearchIndexModel(definition=index_params, name=index_name, type="vectorSearch")
+
+            self.collection.create_search_index(search_index)
+            log.info(f"Created vector search index: {index_name}")
+            self._wait_for_index_ready(index_name)
+
+            # Create regular index on id field for faster lookups
+            self.collection.create_index(self.id_field)
+            log.info(f"Created index on {self.id_field} field")
+
+        except Exception:
+            log.exception(f"Error creating index {index_name}")
+            raise
+
+    def _wait_for_index_ready(self, index_name: str, check_interval: int = 5) -> None:
+        """Wait for index to be ready"""
+        while True:
+            indices = list(self.collection.list_search_indexes())
+            log.debug(f"index status {indices}")
+            if indices and any(idx.get("name") == index_name and idx.get("queryable") for idx in indices):
+                break
+            for idx in indices:
+                if idx.get("name") == index_name and idx.get("status") == "FAILED":
+                    error_msg = f"Index {index_name} failed to build"
+                    raise MongoDBError(error_msg)
+
+            time.sleep(check_interval)
+        log.info(f"Index {index_name} is ready")
+
+    def need_normalize_cosine(self) -> bool:
+        return False
+
+    def insert_embeddings(
+        self,
+        embeddings: list[list[float]],
+        metadata: list[int],
+        **kwargs,
+    ) -> (int, Exception | None):
+        """Insert embeddings into MongoDB"""
+
+        # Prepare documents in bulk
+        documents = [
+            {
+                self.id_field: id_,
+                self.vector_field: embedding,
+            }
+            for id_, embedding in zip(metadata, embeddings, strict=False)
+        ]
+
+        # Use ordered=False for better insert performance
+        try:
+            self.collection.insert_many(documents, ordered=False)
+        except Exception as e:
+            return 0, e
+        return len(documents), None
+
+    def search_embedding(
+        self,
+        query: list[float],
+        k: int = 100,
+        filters: dict | None = None,
+        **kwargs,
+    ) -> list[int]:
+        """Search for similar vectors"""
+        search_params = self.case_config.search_param()
+
+        vector_search = {"queryVector": query, "index": "vector_index", "path": self.vector_field, "limit": k}
+
+        # Add exact search parameter if specified
+        if search_params["exact"]:
+            vector_search["exact"] = True
+        else:
+            # Set numCandidates based on k value and data size
+            # For 50K dataset, use higher multiplier for better recall
+            num_candidates = min(10000, k * search_params["num_candidates_ratio"])
+            vector_search["numCandidates"] = num_candidates
+
+        # Add filter if specified
+        if filters:
+            log.info(f"Applying filter: {filters}")
+            vector_search["filter"] = {
+                "id": {"gte": filters["id"]},
+            }
+        pipeline = [
+            {"$vectorSearch": vector_search},
+            {
+                "$project": {
+                    "_id": 0,
+                    self.id_field: 1,
+                    "score": {"$meta": "vectorSearchScore"},  # Include similarity score
+                }
+            },
+        ]
+
+        results = list(self.collection.aggregate(pipeline))
+        return [doc[self.id_field] for doc in results]
+
+    def optimize(self, data_size: int | None = None) -> None:
+        """MongoDB vector search indexes are self-optimizing"""
+        log.info("optimize for search")
+        self._create_index()
+        self._wait_for_index_ready("vector_index")
+
+    def ready_to_load(self) -> None:
+        """MongoDB is always ready to load"""

vectordb_bench/frontend/components/custom/displaypPrams.py

@@ -3,7 +3,7 @@ def displayParams(st):
         """
 - `Folder Path` - The path to the folder containing all the files. Please ensure that all files in the folder are in the `Parquet` format.
   - Vectors data files: The file must be named `train.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`.
-  - Query test vectors: The file must be named `test.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`.
+  - Query test vectors: The file must be named `test.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`. 
   - Ground truth file: The file must be named `neighbors.parquet` and should have two columns: `id` corresponding to query vectors and `neighbors_id` as an array of `int`.
 
 - `Train File Count` - If the vector file is too large, you can consider splitting it into multiple files. The naming format for the split files should be `train-[index]-of-[file_count].parquet`. For example, `train-01-of-10.parquet` represents the second file (0-indexed) among 10 split files.
@@ -11,3 +11,14 @@ def displayParams(st):
 - `Use Shuffled Data` - If you check this option, the vector data files need to be modified. VectorDBBench will load the data labeled with `shuffle`. For example, use `shuffle_train.parquet` instead of `train.parquet` and `shuffle_train-04-of-10.parquet` instead of `train-04-of-10.parquet`. The `id` column in the shuffled data can be in any order.
 """
     )
+    st.caption(
+        """We recommend limiting the number of test query vectors, like 1,000.""",
+        help="""
+When conducting concurrent query tests, Vdbbench creates a large number of processes. 
+To minimize additional communication overhead during testing, 
+we prepare a complete set of test queries for each process, allowing them to run independently.\n
+However, this means that as the number of concurrent processes increases, 
+the number of copied query vectors also increases significantly, 
+which can place substantial pressure on memory resources.
+""",
+    )

vectordb_bench/frontend/components/run_test/submitTask.py

@@ -1,6 +1,8 @@
 from datetime import datetime
+from vectordb_bench import config
 from vectordb_bench.frontend.config import styles
 from vectordb_bench.interface import benchmark_runner
+from vectordb_bench.models import TaskConfig
 
 
 def submitTask(st, tasks, isAllValid):
@@ -47,16 +49,31 @@ def advancedSettings(st):
     k = container[0].number_input("k", min_value=1, value=100, label_visibility="collapsed")
     container[1].caption("K value for number of nearest neighbors to search")
 
-    return index_already_exists, use_aliyun, k
+    container = st.columns([1, 2])
+    defaultconcurrentInput = ",".join(map(str, config.NUM_CONCURRENCY))
+    concurrentInput = container[0].text_input(
+        "Concurrent Input", value=defaultconcurrentInput, label_visibility="collapsed"
+    )
+    container[1].caption("num of concurrencies for search tests to get max-qps")
+    return index_already_exists, use_aliyun, k, concurrentInput
 
 
-def controlPanel(st, tasks, taskLabel, isAllValid):
-    index_already_exists, use_aliyun, k = advancedSettings(st)
+def controlPanel(st, tasks: list[TaskConfig], taskLabel, isAllValid):
+    index_already_exists, use_aliyun, k, concurrentInput = advancedSettings(st)
 
     def runHandler():
         benchmark_runner.set_drop_old(not index_already_exists)
+
+        try:
+            concurrentInput_list = [int(item.strip()) for item in concurrentInput.split(",")]
+        except ValueError:
+            st.write("please input correct number")
+            return None
+
         for task in tasks:
             task.case_config.k = k
+            task.case_config.concurrency_search_config.num_concurrency = concurrentInput_list
+
         benchmark_runner.set_download_address(use_aliyun)
         benchmark_runner.run(tasks, taskLabel)
 

vectordb_bench/frontend/config/dbCaseConfigs.py

@@ -1041,6 +1041,26 @@ CaseConfigParamInput_NumCandidates_AliES = CaseConfigInput(
 )
 
 
+CaseConfigParamInput_MongoDBQuantizationType = CaseConfigInput(
+    label=CaseConfigParamType.mongodb_quantization_type,
+    inputType=InputType.Option,
+    inputConfig={
+        "options": ["none", "scalar", "binary"],
+    },
+)
+
+
+CaseConfigParamInput_MongoDBNumCandidatesRatio = CaseConfigInput(
+    label=CaseConfigParamType.mongodb_num_candidates_ratio,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 10,
+        "max": 20,
+        "value": 10,
+    },
+)
+
+
 MilvusLoadConfig = [
     CaseConfigParamInput_IndexType,
     CaseConfigParamInput_M,
@@ -1224,6 +1244,14 @@ AliyunElasticsearchPerformanceConfig = [
     CaseConfigParamInput_NumCandidates_AliES,
 ]
 
+MongoDBLoadingConfig = [
+    CaseConfigParamInput_MongoDBQuantizationType,
+]
+MongoDBPerformanceConfig = [
+    CaseConfigParamInput_MongoDBQuantizationType,
+    CaseConfigParamInput_MongoDBNumCandidatesRatio,
+]
+
 CASE_CONFIG_MAP = {
     DB.Milvus: {
         CaseLabel.Load: MilvusLoadConfig,
@@ -1272,4 +1300,8 @@ CASE_CONFIG_MAP = {
         CaseLabel.Load: AliyunOpensearchLoadingConfig,
         CaseLabel.Performance: AliyunOpenSearchPerformanceConfig,
     },
+    DB.MongoDB: {
+        CaseLabel.Load: MongoDBLoadingConfig,
+        CaseLabel.Performance: MongoDBPerformanceConfig,
+    },
 }

vectordb_bench/log_util.py

@@ -1,8 +1,13 @@
 import logging
 from logging import config
+from pathlib import Path
 
 
 def init(log_level: str):
+    # Create logs directory if it doesn't exist
+    log_dir = Path("logs")
+    log_dir.mkdir(exist_ok=True)
+
     log_config = {
         "version": 1,
         "disable_existing_loggers": False,
@@ -24,15 +29,23 @@ def init(log_level: str):
                 "class": "logging.StreamHandler",
                 "formatter": "default",
             },
+            "file": {
+                "class": "logging.handlers.RotatingFileHandler",
+                "formatter": "default",
+                "filename": "logs/vectordb_bench.log",
+                "maxBytes": 10485760,  # 10MB
+                "backupCount": 5,
+                "encoding": "utf8",
+            },
         },
         "loggers": {
             "vectordb_bench": {
-                "handlers": ["console"],
+                "handlers": ["console", "file"],
                 "level": log_level,
                 "propagate": False,
             },
             "no_color": {
-                "handlers": ["no_color_console"],
+                "handlers": ["no_color_console", "file"],
                 "level": log_level,
                 "propagate": False,
             },

vectordb_bench/models.py

@@ -88,6 +88,10 @@ class CaseConfigParamType(Enum):
     numSearchThreads = "num_search_threads"
     maxNumPrefetchDatasets = "max_num_prefetch_datasets"
 
+    # mongodb params
+    mongodb_quantization_type = "quantization"
+    mongodb_num_candidates_ratio = "num_candidates_ratio"
+
 
 class CustomizedCase(BaseModel):
     pass

{vectordb_bench-0.0.21.dist-info → vectordb_bench-0.0.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: vectordb-bench
-Version: 0.0.21
+Version: 0.0.22
 Summary: VectorDBBench is not just an offering of benchmark results for mainstream vector databases and cloud services, it's your go-to tool for the ultimate performance and cost-effectiveness comparison. Designed with ease-of-use in mind, VectorDBBench is devised to help users, even non-professionals, reproduce results or test new systems, making the hunt for the optimal choice amongst a plethora of cloud services and open-source vector databases a breeze.
 Author-email: XuanYang-cn <xuan.yang@zilliz.com>
 Project-URL: repository, https://github.com/zilliztech/VectorDBBench
@@ -21,7 +21,7 @@ Requires-Dist: oss2
 Requires-Dist: psutil
 Requires-Dist: polars
 Requires-Dist: plotly
-Requires-Dist: environs
+Requires-Dist: environs<14.1.0
 Requires-Dist: pydantic<v2
 Requires-Dist: scikit-learn
 Requires-Dist: pymilvus
@@ -73,6 +73,8 @@ Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: aliyun-opensearch
 Requires-Dist: alibabacloud_ha3engine_vector; extra == "aliyun-opensearch"
 Requires-Dist: alibabacloud_searchengine20211025; extra == "aliyun-opensearch"
+Provides-Extra: mongodb
+Requires-Dist: pymongo; extra == "mongodb"
 
 # VectorDBBench: A Benchmark Tool for VectorDB
 
@@ -89,6 +91,8 @@ Closely mimicking real-world production environments, we've set up diverse testi
 
 Prepare to delve into the world of VectorDBBench, and let it guide you in uncovering your perfect vector database match.  
 
+VectorDBBench is sponsered by Zilliz，the leading opensource vectorDB company behind Milvus. Choose smarter with VectorDBBench- start your free test on [zilliz cloud](https://zilliz.com/) today!
+
 **Leaderboard:** https://zilliz.com/benchmark
 ## Quick Start
 ### Prerequirement
@@ -128,6 +132,7 @@ All the database client supported
 | chromadb                 | `pip install vectordb-bench[chromadb]`      |
 | awsopensearch            | `pip install vectordb-bench[opensearch]` |
 | aliyun_opensearch        | `pip install vectordb-bench[aliyun_opensearch]` |
+| mongodb                  | `pip install vectordb-bench[mongodb]`       |
 
 ### Run
 
@@ -228,6 +233,47 @@ Options:
                                   with-gt]
   --help                          Show this message and exit.
 ```
+
+### Run awsopensearch from command line
+
+```shell
+vectordbbench awsopensearch --db-label awsopensearch \
+--m 16 --ef-construction 256 \
+--host search-vector-db-prod-h4f6m4of6x7yp2rz7gdmots7w4.us-west-2.es.amazonaws.com --port 443 \
+--user vector --password '<password>' \
+--case-type Performance1536D5M --num-insert-workers 10  \
+--skip-load --num-concurrency 75
+```
+
+To list the options for awsopensearch, execute `vectordbbench awsopensearch --help`
+
+```text
+$ vectordbbench awsopensearch --help
+Usage: vectordbbench awsopensearch [OPTIONS]
+
+Options:
+  # Sharding and Replication
+  --number-of-shards INTEGER      Number of primary shards for the index
+  --number-of-replicas INTEGER    Number of replica copies for each primary
+                                  shard
+  # Indexing Performance                              
+  --index-thread-qty INTEGER      Thread count for native engine indexing
+  --index-thread-qty-during-force-merge INTEGER
+                                  Thread count during force merge operations
+  --number-of-indexing-clients INTEGER
+                                  Number of concurrent indexing clients
+  # Index Management
+  --number-of-segments INTEGER    Target number of segments after merging
+  --refresh-interval TEXT         How often to make new data available for
+                                  search
+  --force-merge-enabled BOOLEAN   Whether to perform force merge operation
+  --flush-threshold-size TEXT     Size threshold for flushing the transaction
+                                  log
+  # Memory Management
+  --cb-threshold TEXT             k-NN Memory circuit breaker threshold
+
+  --help                          Show this message and exit.```
+
 #### Using a configuration file.
 
 The vectordbbench command can optionally read some or all the options from a yaml formatted configuration file.
@@ -394,6 +440,13 @@ We have strict requirements for the data set format, please follow them.
 - `Folder Path` - The path to the folder containing all the files. Please ensure that all files in the folder are in the `Parquet` format.
   - Vectors data files: The file must be named `train.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`.
   - Query test vectors: The file must be named `test.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`.
+    - We recommend limiting the number of test query vectors, like 1,000.
+    When conducting concurrent query tests, Vdbbench creates a large number of processes. 
+    To minimize additional communication overhead during testing, 
+    we prepare a complete set of test queries for each process, allowing them to run independently.
+    However, this means that as the number of concurrent processes increases, 
+    the number of copied query vectors also increases significantly, 
+    which can place substantial pressure on memory resources.
   - Ground truth file: The file must be named `neighbors.parquet` and should have two columns: `id` corresponding to query vectors and `neighbors_id` as an array of `int`.
 
 - `Train File Count` - If the vector file is too large, you can consider splitting it into multiple files. The naming format for the split files should be `train-[index]-of-[file_count].parquet`. For example, `train-01-of-10.parquet` represents the second file (0-indexed) among 10 split files.

{vectordb_bench-0.0.21.dist-info → vectordb_bench-0.0.22.dist-info}/RECORD

@@ -2,9 +2,9 @@ vectordb_bench/__init__.py,sha256=d5psAfISw9F6PFL2xPlSYUKKFDw7ifQm7g3LWC8_yUA,23
 vectordb_bench/__main__.py,sha256=cyYbVSU-zA1AgzneGKcRRuzR4ftRDr9sIi9Ei9NZnhI,858
 vectordb_bench/base.py,sha256=AgavIF0P9ku_RmCRk1KKziba-wI4ZpA2aJvjJzNhRSs,129
 vectordb_bench/interface.py,sha256=XaCjTgUeI17uVjsgOauPeVlkvnkuCyQOWyOaWhrgCt8,9811
-vectordb_bench/log_util.py,sha256=hOdK0TnrcpYZOrRZoBslievXSW8qtTvLvube43rxbVc,2776
+vectordb_bench/log_util.py,sha256=wDNaU_JBBOfKi_Z4vq7LDa0kOlLjoNNzDX3VZQn_Dxo,3239
 vectordb_bench/metric.py,sha256=pj-AxQHyIRHTaJY-wTIkTbC6TqEqMzt3kcEmMWEv71w,2063
-vectordb_bench/models.py,sha256=5N4-0lJLWpoR6NnzX4ONuH7vyi4nRFFuNS0q9jQ4cgM,11023
+vectordb_bench/models.py,sha256=1G6GDTtP0TbYT2Qhw_cJ84kx_Wkn4gq6V_EUVpCJW3Y,11150
 vectordb_bench/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vectordb_bench/backend/assembler.py,sha256=6GInRT7yBgfTaIPmo-XMkYX4pA8PJQmjMQInynwaunE,2047
 vectordb_bench/backend/cases.py,sha256=obDdY6g3p9Z2fog7qDwLLDuRMwo3LGQKMHsP66QZd2M,16296
@@ -13,7 +13,7 @@ vectordb_bench/backend/dataset.py,sha256=V4OKPt23v0kmdvgJwDr_R2fLJv3lXLZEii992cE
 vectordb_bench/backend/result_collector.py,sha256=mpROVdZ-HChKBVyMV5TZ5v7YGRb69bvfT7Gezn5F5sY,819
 vectordb_bench/backend/task_runner.py,sha256=vlaXB0_25-G9w1Lj-F0SrvJzhXT7ceDWGIb2aKRXukU,11488
 vectordb_bench/backend/utils.py,sha256=R6THuJdZhiQYSSJTqv0Uegl2B20taV_QjwvFrun2yxE,1949
-vectordb_bench/backend/clients/__init__.py,sha256=YvXoI8CS69WOTLl5n858xkRReeqiiRFBETZAAnD14qg,8212
+vectordb_bench/backend/clients/__init__.py,sha256=ncF4H4foep04EdpzQHQ6zqDxTL1cSvUmglt-v2E966g,8595
 vectordb_bench/backend/clients/api.py,sha256=uQaX_FiMFlD3z_91awUzB-qtBkvyDsMKE8ks5bBgJSY,6233
 vectordb_bench/backend/clients/aliyun_elasticsearch/aliyun_elasticsearch.py,sha256=7yPYaWoHeHNxDMtpReGXsdEPFD1e4vQblFor7TmLq5o,770
 vectordb_bench/backend/clients/aliyun_elasticsearch/config.py,sha256=d9RCgfCgauKvy6z9ig_wBormgwiGtkh8POyoHloHnJA,505
@@ -22,9 +22,9 @@ vectordb_bench/backend/clients/aliyun_opensearch/config.py,sha256=KSiuRu-p7oL2PE
 vectordb_bench/backend/clients/alloydb/alloydb.py,sha256=E24hxCUgpBCRiScdcS_iBk8n0wngUgVg8qujOWiUhw0,13009
 vectordb_bench/backend/clients/alloydb/cli.py,sha256=G6Q0WApoDXDG_pqmK2lEKFIvKB8qAsZFPM8TfsURydE,5086
 vectordb_bench/backend/clients/alloydb/config.py,sha256=PJs2wIJqwcG6UJ3T8R7Pi3xTMBfxTZiNkcWyhtHv5dc,5313
-vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py,sha256=PAFP5W6k0GxbpvvIFs8u3pMi7x-sLw9cm4mUsM2agws,7541
-vectordb_bench/backend/clients/aws_opensearch/cli.py,sha256=845dUYcD5m9j9FNOCXAspgCkvOKWWJXt2k87I55Odbs,1301
-vectordb_bench/backend/clients/aws_opensearch/config.py,sha256=XsQBKGBPR0lqH3XrQpijwgt9Tfb6KBVg35x2L9LQQMw,1881
+vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py,sha256=iRtPmHZoVTpQ-3Q90nE70zy_XsklGlSSNgBOgeAtVzU,10047
+vectordb_bench/backend/clients/aws_opensearch/cli.py,sha256=G086STCoaTBkz2J5Qt42bnyhmcYbhl6XxTaLfeirkXQ,4065
+vectordb_bench/backend/clients/aws_opensearch/config.py,sha256=9meXQUOVFlk3UOAhvBhaghNm7TasDsA6-fXOY8C9gzU,2295
 vectordb_bench/backend/clients/aws_opensearch/run.py,sha256=Ry5aAlielWjq0hx7LnbdShfOwzZhz3Gq9WYu5U43x9s,5001
 vectordb_bench/backend/clients/chroma/chroma.py,sha256=TGsmAnG5I3bbIjJ5L7ktke6fD8lOrx56Wt2tMCb3dY8,3609
 vectordb_bench/backend/clients/chroma/config.py,sha256=8nXpPdecQ5HrNqcsQwAVgacSz6uLgI-BI7v4tB8CeDk,347
@@ -36,6 +36,8 @@ vectordb_bench/backend/clients/memorydb/memorydb.py,sha256=WrZhDYJqpwN173sk2lmPn
 vectordb_bench/backend/clients/milvus/cli.py,sha256=xGvYYKOAs32vz78oB5Ks_xnWIMzcl_f7TPEPRk94FeQ,8895
 vectordb_bench/backend/clients/milvus/config.py,sha256=oFZ5VG5UHws161M1cYmMr2b9NSEoqwwst998T59QGQo,7520
 vectordb_bench/backend/clients/milvus/milvus.py,sha256=xdVVjMnBzD5KGJ7iUB-B3SuTL4JDW1UD15QBevExMLw,6862
+vectordb_bench/backend/clients/mongodb/config.py,sha256=7DZCh0bjPiqJW2luPypfpNeGfvKxVC4mdHLqgcjF1hA,1745
+vectordb_bench/backend/clients/mongodb/mongodb.py,sha256=ts2gpAzUTarpkfMFnM5ANi6T-xvcjS8kc4-apPt9jug,7225
 vectordb_bench/backend/clients/pgdiskann/cli.py,sha256=o5ddAp1Be2TOnm8Wh9IyIWUxdnw5N6v92Ms1s6CEwBo,3135
 vectordb_bench/backend/clients/pgdiskann/config.py,sha256=DBsVgLn4edl-irSlP_GV7KW-8jFemns_ujR_CuVnQtE,4412
 vectordb_bench/backend/clients/pgdiskann/pgdiskann.py,sha256=Z8K74Y6uMi6q8gnnD68doBxc5pWBSpRnNLDhlifseH4,12299
@@ -88,7 +90,7 @@ vectordb_bench/frontend/components/check_results/priceTable.py,sha256=K3NmlNKAb-
 vectordb_bench/frontend/components/check_results/stPageConfig.py,sha256=czkqr9NC3UQAxiz8KSCZC8cPmgSnFUhI2lOLHXfuMxo,432
 vectordb_bench/frontend/components/concurrent/charts.py,sha256=00WI8wxIdHAhnpmFJLd03n5U3LbowmeY4swVbGNzyYg,2874
 vectordb_bench/frontend/components/custom/displayCustomCase.py,sha256=aIWKFm13-EPG2XlJ3PWc2znR6q8A5FR93D5ZkGGncrM,1641
-vectordb_bench/frontend/components/custom/displaypPrams.py,sha256=GNs-awcbYAyOTgmnsFl_EmwjJHhD8EoN86a1-iQfCnc,1335
+vectordb_bench/frontend/components/custom/displaypPrams.py,sha256=mwm74_86YYRbpJ1Hz2Dba0eKvyzkK0DM7uhjBDFoElU,1910
 vectordb_bench/frontend/components/custom/getCustomConfig.py,sha256=tSPI2DPJSNxlArLcO5Kf9nhpIBc0_YE2QD9-1cbaLus,1031
 vectordb_bench/frontend/components/custom/initStyle.py,sha256=ortsoUNqH-vVq9ECiw80PnBEcIaUwxR1AQ65DSkBhGs,434
 vectordb_bench/frontend/components/get_results/saveAsImage.py,sha256=POaFiwKoCGqrY-zhanWC7-tubE64bV_JjqI4lgIuMts,1459
@@ -99,9 +101,9 @@ vectordb_bench/frontend/components/run_test/dbSelector.py,sha256=hzMEIL1DzvpP8xk
 vectordb_bench/frontend/components/run_test/generateTasks.py,sha256=3y8NHtWJMNqoP2SvoWuR7kj84g0OEg68IULebimzz7E,741
 vectordb_bench/frontend/components/run_test/hideSidebar.py,sha256=vb5kzIMmbMqWX67qFEHek21X4sGO_tPyn_uPqUEtp3Q,234
 vectordb_bench/frontend/components/run_test/initStyle.py,sha256=osPUgfFfH7rRlVNHSMumvmZxvKWlLxmZiNqgnMiUJEU,723
-vectordb_bench/frontend/components/run_test/submitTask.py,sha256=8Ka8n7eviZi56BXfcrpsUqhLcYUHgyif0zzQ2w2hHMw,3328
+vectordb_bench/frontend/components/run_test/submitTask.py,sha256=VZjkopkCBNhqLwGqsoM0hbPEeF6Q5UOQcdFUaegerxc,4094
 vectordb_bench/frontend/components/tables/data.py,sha256=5DdnC64BB7Aj2z9acht2atsPB4NabzQCZKALfIUnqtQ,1233
-vectordb_bench/frontend/config/dbCaseConfigs.py,sha256=cjFLhmHaNJKhPsAplKnjIO7ypZnlSw7S83N-WXI_gUQ,37781
+vectordb_bench/frontend/config/dbCaseConfigs.py,sha256=vHYdD3Tzi4NRD5ckJfkciIqZcMZY_Qslm_sUolKLxro,38597
 vectordb_bench/frontend/config/dbPrices.py,sha256=10aBKjVcEg8y7TPSda28opmBM1KmXNrvbU9WM_BsZcE,176
 vectordb_bench/frontend/config/styles.py,sha256=E2PmwmiewxBKJJ59hQ4ZXatqg8QTN-Z53JlsvWMHM2M,2291
 vectordb_bench/frontend/pages/concurrent.py,sha256=bvoSafRSIsRzBQkI3uBwwrdg8jnhRUQG-epZbrJhGiE,2082
@@ -127,9 +129,9 @@ vectordb_bench/results/WeaviateCloud/result_20230808_standard_weaviatecloud.json
 vectordb_bench/results/ZillizCloud/result_20230727_standard_zillizcloud.json,sha256=-Mdm4By65XDRCrmVOCF8yQXjcZtH4Xo4shcjoDoBUKU,18293
 vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json,sha256=77XlHT5zM_K7mG5HfDQKwXZnSCuR37VUbt6-P3J_amI,15737
 vectordb_bench/results/ZillizCloud/result_20240105_standard_202401_zillizcloud.json,sha256=TualfJ0664Hs-vdIW68bdkqAEYyzotXmu2P0yIN-GHk,42526
-vectordb_bench-0.0.21.dist-info/LICENSE,sha256=HXbxhrb5u5SegVzeLNF_voVgRsJMavcLaOmD1N0lZkM,1067
-vectordb_bench-0.0.21.dist-info/METADATA,sha256=SDCFG-7cwQRoLe3mrvjCQNslK1Ju8aw0VQ_Kc8408hw,34577
-vectordb_bench-0.0.21.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-vectordb_bench-0.0.21.dist-info/entry_points.txt,sha256=Qzw6gVx96ui8esG21H6yHsI6nboEohRmV424TYhQNrA,113
-vectordb_bench-0.0.21.dist-info/top_level.txt,sha256=jnhZFZAuKX1J60yt-XOeBZ__ctiZMvoC_s0RFq29lpM,15
-vectordb_bench-0.0.21.dist-info/RECORD,,
+vectordb_bench-0.0.22.dist-info/LICENSE,sha256=HXbxhrb5u5SegVzeLNF_voVgRsJMavcLaOmD1N0lZkM,1067
+vectordb_bench-0.0.22.dist-info/METADATA,sha256=fSJ2cTKb1oLQcXYVQc8cTHoMkEqOelusy0a_VEPuvPo,37166
+vectordb_bench-0.0.22.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+vectordb_bench-0.0.22.dist-info/entry_points.txt,sha256=Qzw6gVx96ui8esG21H6yHsI6nboEohRmV424TYhQNrA,113
+vectordb_bench-0.0.22.dist-info/top_level.txt,sha256=jnhZFZAuKX1J60yt-XOeBZ__ctiZMvoC_s0RFq29lpM,15
+vectordb_bench-0.0.22.dist-info/RECORD,,