vectordb-bench 0.0.18__py3-none-any.whl → 0.0.19__py3-none-any.whl

vectordb_bench/backend/cases.py

@@ -289,7 +289,7 @@ class Performance1536D50K(PerformanceCase):
     description: str = """This case tests the search performance of a vector database with a medium 50K dataset (<b>OpenAI 50K vectors</b>, 1536 dimensions), at varying parallel levels.
 Results will show index building time, recall, and maximum QPS."""
     load_timeout: float | int = 3600
-    optimize_timeout: float | int | None = 15 * 60
+    optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_DEFAULT
 
 
 def metric_type_map(s: str) -> MetricType:

vectordb_bench/backend/clients/__init__.py

@@ -37,7 +37,9 @@ class DB(Enum):
     MemoryDB = "MemoryDB"
     Chroma = "Chroma"
     AWSOpenSearch = "OpenSearch"
+    AliyunElasticsearch = "AliyunElasticsearch"
     Test = "test"
+    AliyunOpenSearch = "AliyunOpenSearch"
 
 
     @property
@@ -103,6 +105,14 @@ class DB(Enum):
             from .alloydb.alloydb import AlloyDB
             return AlloyDB
 
+        if self == DB.AliyunElasticsearch:
+            from .aliyun_elasticsearch.aliyun_elasticsearch import AliyunElasticsearch
+            return AliyunElasticsearch
+
+        if self == DB.AliyunOpenSearch:
+            from .aliyun_opensearch.aliyun_opensearch import AliyunOpenSearch
+            return AliyunOpenSearch
+
     @property
     def config_cls(self) -> Type[DBConfig]:
         """Import while in use"""
@@ -166,6 +176,14 @@ class DB(Enum):
             from .alloydb.config import AlloyDBConfig
             return AlloyDBConfig
 
+        if self == DB.AliyunElasticsearch:
+            from .aliyun_elasticsearch.config import AliyunElasticsearchConfig
+            return AliyunElasticsearchConfig
+
+        if self == DB.AliyunOpenSearch:
+            from .aliyun_opensearch.config import AliyunOpenSearchConfig
+            return AliyunOpenSearchConfig
+
     def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseConfig]:
         if self == DB.Milvus:
             from .milvus.config import _milvus_case_config
@@ -211,6 +229,14 @@ class DB(Enum):
             from .alloydb.config import _alloydb_case_config
             return _alloydb_case_config.get(index_type)
 
+        if self == DB.AliyunElasticsearch:
+            from .elastic_cloud.config import ElasticCloudIndexConfig
+            return ElasticCloudIndexConfig
+
+        if self == DB.AliyunOpenSearch:
+            from .aliyun_opensearch.config import AliyunOpenSearchIndexConfig
+            return AliyunOpenSearchIndexConfig
+
         # DB.Pinecone, DB.Chroma, DB.Redis
         return EmptyDBCaseConfig
 

vectordb_bench/backend/clients/aliyun_elasticsearch/aliyun_elasticsearch.py

@@ -0,0 +1,27 @@
+from ..elastic_cloud.elastic_cloud import ElasticCloud
+from ..elastic_cloud.config import ElasticCloudIndexConfig
+
+
+class AliyunElasticsearch(ElasticCloud):
+    def __init__(
+        self,
+        dim: int,
+        db_config: dict,
+        db_case_config: ElasticCloudIndexConfig,
+        indice: str = "vdb_bench_indice",  # must be lowercase
+        id_col_name: str = "id",
+        vector_col_name: str = "vector",
+        drop_old: bool = False,
+        **kwargs,
+    ):
+        super().__init__(
+            dim=dim,
+            db_config=db_config,
+            db_case_config=db_case_config,
+            indice=indice,
+            id_col_name=id_col_name,
+            vector_col_name=vector_col_name,
+            drop_old=drop_old,
+            **kwargs,
+        )
+

vectordb_bench/backend/clients/aliyun_elasticsearch/config.py

@@ -0,0 +1,19 @@
+from enum import Enum
+from pydantic import SecretStr, BaseModel
+
+from ..api import DBConfig, DBCaseConfig, MetricType, IndexType
+
+
+class AliyunElasticsearchConfig(DBConfig, BaseModel):
+    #: Protocol in use to connect to the node
+    scheme: str = "http"
+    host: str = ""
+    port: int = 9200
+    user: str = "elastic"
+    password: SecretStr
+
+    def to_dict(self) -> dict:
+        return {
+            "hosts": [{'scheme': self.scheme, 'host': self.host, 'port': self.port}],
+            "basic_auth": (self.user, self.password.get_secret_value()),
+        }

vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py

@@ -0,0 +1,304 @@
+import json
+import logging
+from contextlib import contextmanager
+import time
+
+from alibabacloud_ha3engine_vector.models import QueryRequest
+
+from ..api import VectorDB, MetricType
+from .config import AliyunOpenSearchIndexConfig
+
+from alibabacloud_searchengine20211025.client import Client as searchengineClient
+from alibabacloud_searchengine20211025 import models as searchengine_models
+from alibabacloud_tea_openapi import models as open_api_models
+from alibabacloud_ha3engine_vector import models, client
+
+log = logging.getLogger(__name__)
+
+ALIYUN_OPENSEARCH_MAX_SIZE_PER_BATCH = 2 * 1024 * 1024  # 2MB
+ALIYUN_OPENSEARCH_MAX_NUM_PER_BATCH = 100
+
+class AliyunOpenSearch(VectorDB):
+    def __init__(
+            self,
+            dim: int,
+            db_config: dict,
+            db_case_config: AliyunOpenSearchIndexConfig,
+            collection_name: str = "VectorDBBenchCollection",
+            drop_old: bool = False,
+            **kwargs,
+    ):
+        self.control_client = None
+        self.dim = dim
+        self.db_config = db_config
+        self.case_config = db_case_config
+        self.collection_name = collection_name
+        self.instance_id = db_config["host"].split(".")[0].replace("http://", "").replace("https://", "")
+
+        self._primary_field = "id"
+        self._scalar_field = "int_id"
+        self._vector_field = "vector"
+        self._index_name = "vector_idx"
+
+        self.batch_size = int(
+            min(ALIYUN_OPENSEARCH_MAX_SIZE_PER_BATCH / (dim * 25), ALIYUN_OPENSEARCH_MAX_NUM_PER_BATCH)
+        )
+
+        log.info(f"Aliyun_OpenSearch client config: {self.db_config}")
+        control_config = open_api_models.Config(
+            access_key_id=self.db_config["ak"],
+            access_key_secret=self.db_config["sk"],
+            endpoint=self.db_config["control_host"]
+        )
+        self.control_client = searchengineClient(control_config)
+
+        if drop_old:
+            log.info(f"aliyun_OpenSearch client drop old index: {self.collection_name}")
+            if self._index_exists(self.control_client):
+                self._modify_index(self.control_client)
+            else:
+                self._create_index(self.control_client)
+
+    def _create_index(self, client: searchengineClient):
+        create_table_request = searchengine_models.CreateTableRequest()
+        create_table_request.name = self.collection_name
+        create_table_request.primary_key = self._primary_field
+        create_table_request.partition_count = 1
+        create_table_request.field_schema = {
+            self._primary_field: "INT64",
+            self._vector_field: "MULTI_FLOAT",
+            self._scalar_field: "INT64"
+        }
+        vector_index = searchengine_models.ModifyTableRequestVectorIndex()
+        vector_index.index_name = self._index_name
+        vector_index.dimension = self.dim
+        vector_index.distance_type = self.case_config.distance_type()
+        vector_index.vector_field = self._vector_field
+        vector_index.vector_index_type = "HNSW"
+
+        advance_params = searchengine_models.ModifyTableRequestVectorIndexAdvanceParams()
+        advance_params.build_index_params = "{\"proxima.hnsw.builder.max_neighbor_count\":" + str(self.case_config.M) + ",\"proxima.hnsw.builder.efconstruction\":" + str(self.case_config.efConstruction) + ",\"proxima.hnsw.builder.enable_adsampling\":true,\"proxima.hnsw.builder.slack_pruning_factor\":1.1,\"proxima.hnsw.builder.thread_count\":16}"
+        advance_params.search_index_params = "{\"proxima.hnsw.searcher.ef\":400,\"proxima.hnsw.searcher.dynamic_termination.prob_threshold\":0.7}"
+        vector_index.advance_params = advance_params
+        create_table_request.vector_index = [vector_index]
+
+        try:
+            response = client.create_table(self.instance_id, create_table_request)
+            log.info(f"create table success: {response.body}")
+        except Exception as error:
+            log.info(error.message)
+            log.info(error.data.get("Recommend"))
+            log.info(f"Failed to create index: error: {str(error)}")
+            raise error from None
+
+        # check if index create success
+        self._active_index(client)
+
+    # check if index create success
+    def _active_index(self, client: searchengineClient) -> None:
+        retry_times = 0
+        while True:
+            time.sleep(10)
+            log.info(f"begin to {retry_times} times get table")
+            retry_times += 1
+            response = client.get_table(self.instance_id, self.collection_name)
+            if response.body.result.status == 'IN_USE':
+                log.info(f"{self.collection_name} table begin to use.")
+                return
+
+    def _index_exists(self, client: searchengineClient) -> bool:
+        try:
+            client.get_table(self.instance_id, self.collection_name)
+            return True
+        except Exception as error:
+            log.info(f'get table from searchengine error')
+            log.info(error.message)
+            return False
+
+    # check if index build success, Insert the embeddings to the vector database after index build success
+    def _index_build_success(self, client: searchengineClient) -> None:
+        log.info(f"begin to check if table build success.")
+        time.sleep(50)
+
+        retry_times = 0
+        while True:
+            time.sleep(10)
+            log.info(f"begin to {retry_times} times get table fsm")
+            retry_times += 1
+            request = searchengine_models.ListTasksRequest()
+            request.start = (int(time.time()) - 3600) * 1000
+            request.end = int(time.time()) * 1000
+            response = client.list_tasks(self.instance_id, request)
+            fsms = response.body.result
+            cur_fsm = None
+            for fsm in fsms:
+                if fsm["type"] != "datasource_flow_fsm":
+                    continue
+                if self.collection_name not in fsm["fsmId"]:
+                    continue
+                cur_fsm = fsm
+                break
+            if cur_fsm is None:
+                print("no build index fsm")
+                return
+            if "success" == cur_fsm["status"]:
+                return
+
+    def _modify_index(self, client: searchengineClient) -> None:
+        # check if index create success
+        self._active_index(client)
+
+        modify_table_request = searchengine_models.ModifyTableRequest()
+        modify_table_request.partition_count = 1
+        modify_table_request.primary_key = self._primary_field
+        modify_table_request.field_schema = {
+            self._primary_field: "INT64",
+            self._vector_field: "MULTI_FLOAT",
+            self._scalar_field: "INT64"
+        }
+        vector_index = searchengine_models.ModifyTableRequestVectorIndex()
+        vector_index.index_name = self._index_name
+        vector_index.dimension = self.dim
+        vector_index.distance_type = self.case_config.distance_type()
+        vector_index.vector_field = self._vector_field
+        vector_index.vector_index_type = "HNSW"
+        advance_params = searchengine_models.ModifyTableRequestVectorIndexAdvanceParams()
+        advance_params.build_index_params = "{\"proxima.hnsw.builder.max_neighbor_count\":" + str(self.case_config.M) + ",\"proxima.hnsw.builder.efconstruction\":" + str(self.case_config.efConstruction) + ",\"proxima.hnsw.builder.enable_adsampling\":true,\"proxima.hnsw.builder.slack_pruning_factor\":1.1,\"proxima.hnsw.builder.thread_count\":16}"
+        advance_params.search_index_params = "{\"proxima.hnsw.searcher.ef\":400,\"proxima.hnsw.searcher.dynamic_termination.prob_threshold\":0.7}"
+        vector_index.advance_params = advance_params
+
+        modify_table_request.vector_index = [vector_index]
+
+        try:
+            response = client.modify_table(self.instance_id, self.collection_name, modify_table_request)
+            log.info(f"modify table success: {response.body}")
+        except Exception as error:
+            log.info(error.message)
+            log.info(error.data.get("Recommend"))
+            log.info(f"Failed to modify index: error: {str(error)}")
+            raise error from None
+
+        # check if modify index & delete data fsm success
+        self._index_build_success(client)
+
+    # get collection records total count
+    def _get_total_count(self):
+        try:
+            response = self.client.stats(self.collection_name)
+            body = json.loads(response.body)
+            log.info(f"stats info: {response.body}")
+
+            if "result" in body and "totalDocCount" in body.get("result"):
+                return body.get("result").get("totalDocCount")
+            else:
+                return 0
+        except Exception as e:
+            print(f"Error querying index: {e}")
+            return 0
+
+    @contextmanager
+    def init(self) -> None:
+        """connect to aliyun opensearch"""
+        config = models.Config(
+            endpoint=self.db_config["host"],
+            protocol="http",
+            access_user_name=self.db_config["user"],
+            access_pass_word=self.db_config["password"]
+        )
+
+        self.client = client.Client(config)
+
+        yield
+        # self.client.transport.close()
+        self.client = None
+        del self.client
+
+    def insert_embeddings(
+            self,
+            embeddings: list[list[float]],
+            metadata: list[int],
+            **kwargs,
+    ) -> tuple[int, Exception]:
+        """Insert the embeddings to the opensearch."""
+        assert self.client is not None, "should self.init() first"
+        assert len(embeddings) == len(metadata)
+        insert_count = 0
+
+        try:
+            for batch_start_offset in range(0, len(embeddings), self.batch_size):
+                batch_end_offset = min(
+                    batch_start_offset + self.batch_size, len(embeddings)
+                )
+                documents = []
+                for i in range(batch_start_offset, batch_end_offset):
+                    documentFields = {
+                        self._primary_field: metadata[i],
+                        self._vector_field: embeddings[i],
+                        self._scalar_field: metadata[i],
+                        "ops_build_channel": "inc"
+                    }
+                    document = {
+                        "fields": documentFields,
+                        "cmd": "add"
+                    }
+                    documents.append(document)
+
+                pushDocumentsRequest = models.PushDocumentsRequest({}, documents)
+                self.client.push_documents(self.collection_name, self._primary_field, pushDocumentsRequest)
+                insert_count += batch_end_offset - batch_start_offset
+        except Exception as e:
+            log.info(f"Failed to insert data: {e}")
+            return (insert_count, e)
+        return (insert_count, None)
+
+    def search_embedding(
+            self,
+            query: list[float],
+            k: int = 100,
+            filters: dict | None = None,
+    ) -> list[int]:
+        assert self.client is not None, "should self.init() first"
+        search_params = "{\"proxima.hnsw.searcher.ef\":"+ str(self.case_config.ef_search) +"}"
+
+        os_filter = f"{self._scalar_field} {filters.get('metadata')}" if filters else ""
+
+        try:
+            request = QueryRequest(table_name=self.collection_name,
+                                   vector=query,
+                                   top_k=k,
+                                   search_params=search_params, filter=os_filter)
+            result = self.client.query(request)
+        except Exception as e:
+            log.info(f"Error querying index: {e}")
+            raise e
+        res = json.loads(result.body)
+        id_res = [one_res["id"] for one_res in res["result"]]
+        return id_res
+
+    def need_normalize_cosine(self) -> bool:
+        """Wheather this database need to normalize dataset to support COSINE"""
+        if self.case_config.metric_type == MetricType.COSINE:
+            log.info(f"cosine dataset need normalize.")
+            return True
+
+        return False
+
+    def optimize(self):
+        pass
+
+    def optimize_with_size(self, data_size: int):
+        log.info(f"optimize count: {data_size}")
+        retry_times = 0
+        while True:
+            time.sleep(10)
+            log.info(f"begin to {retry_times} times get optimize table")
+            retry_times += 1
+            total_count = self._get_total_count()
+            # check if the data is inserted
+            if total_count == data_size:
+                log.info(f"optimize table finish.")
+                return
+
+    def ready_to_load(self):
+        """ready_to_load will be called before load in load cases."""
+        pass

vectordb_bench/backend/clients/aliyun_opensearch/config.py

@@ -0,0 +1,48 @@
+import logging
+from enum import Enum
+from pydantic import SecretStr, BaseModel
+
+from ..api import DBConfig, DBCaseConfig, MetricType, IndexType
+
+log = logging.getLogger(__name__)
+
+
+class AliyunOpenSearchConfig(DBConfig, BaseModel):
+    host: str = ""
+    user: str = ""
+    password: SecretStr = ""
+
+    ak: str = ""
+    sk: SecretStr = ""
+    control_host: str = "searchengine.cn-hangzhou.aliyuncs.com"
+
+    def to_dict(self) -> dict:
+        return {
+            "host": self.host,
+            "user": self.user,
+            "password": self.password.get_secret_value(),
+            "ak": self.ak,
+            "sk": self.sk.get_secret_value(),
+            "control_host": self.control_host,
+        }
+
+class AliyunOpenSearchIndexConfig(BaseModel, DBCaseConfig):
+    metric_type: MetricType = MetricType.L2
+    efConstruction: int = 500
+    M: int = 100
+    ef_search: int = 40
+
+    def distance_type(self) -> str:
+        if self.metric_type == MetricType.L2:
+            return "SquaredEuclidean"
+        elif self.metric_type == MetricType.IP:
+            return "InnerProduct"
+        elif self.metric_type == MetricType.COSINE:
+            return "InnerProduct"
+        return "SquaredEuclidean"
+
+    def index_param(self) -> dict:
+        return {}
+
+    def search_param(self) -> dict:
+        return {}

vectordb_bench/backend/clients/alloydb/cli.py

@@ -106,7 +106,7 @@ class AlloyDBScaNNTypedDict(AlloyDBTypedDict):
         int,
         click.option(
             "--max-num-levels",
-            type=click.Choice([1, 2]),
+            type=click.Choice(["1", "2"]),
             help="Maximum number of levels",
             default=1
         )

vectordb_bench/backend/clients/api.py

@@ -204,6 +204,9 @@ class VectorDB(ABC):
         """
         raise NotImplementedError
 
+    def optimize_with_size(self, data_size: int):
+        self.optimize()
+
     # TODO: remove
     @abstractmethod
     def ready_to_load(self):

vectordb_bench/backend/clients/milvus/cli.py

@@ -1,4 +1,4 @@
-from typing import Annotated, TypedDict, Unpack
+from typing import Annotated, TypedDict, Unpack, Optional
 
 import click
 from pydantic import SecretStr
@@ -21,6 +21,12 @@ class MilvusTypedDict(TypedDict):
     uri: Annotated[
         str, click.option("--uri", type=str, help="uri connection string", required=True)
     ]
+    user_name: Annotated[
+        Optional[str], click.option("--user-name", type=str, help="Db username", required=False)
+    ]
+    password: Annotated[
+        Optional[str], click.option("--password", type=str, help="Db password", required=False)
+    ]
 
 
 class MilvusAutoIndexTypedDict(CommonTypedDict, MilvusTypedDict):
@@ -37,6 +43,8 @@ def MilvusAutoIndex(**parameters: Unpack[MilvusAutoIndexTypedDict]):
         db_config=MilvusConfig(
             db_label=parameters["db_label"],
             uri=SecretStr(parameters["uri"]),
+            user=parameters["user_name"],
+            password=SecretStr(parameters["password"]),
         ),
         db_case_config=AutoIndexConfig(),
         **parameters,
@@ -53,6 +61,8 @@ def MilvusFlat(**parameters: Unpack[MilvusAutoIndexTypedDict]):
         db_config=MilvusConfig(
             db_label=parameters["db_label"],
             uri=SecretStr(parameters["uri"]),
+            user=parameters["user_name"],
+            password=SecretStr(parameters["password"]),
         ),
         db_case_config=FLATConfig(),
         **parameters,
@@ -73,6 +83,8 @@ def MilvusHNSW(**parameters: Unpack[MilvusHNSWTypedDict]):
         db_config=MilvusConfig(
             db_label=parameters["db_label"],
             uri=SecretStr(parameters["uri"]),
+            user=parameters["user_name"],
+            password=SecretStr(parameters["password"]) if parameters["password"] else None,
         ),
         db_case_config=HNSWConfig(
             M=parameters["m"],
@@ -97,6 +109,8 @@ def MilvusIVFFlat(**parameters: Unpack[MilvusIVFFlatTypedDict]):
         db_config=MilvusConfig(
             db_label=parameters["db_label"],
             uri=SecretStr(parameters["uri"]),
+            user=parameters["user_name"],
+            password=SecretStr(parameters["password"]),
         ),
         db_case_config=IVFFlatConfig(
             nlist=parameters["nlist"],
@@ -116,6 +130,8 @@ def MilvusIVFSQ8(**parameters: Unpack[MilvusIVFFlatTypedDict]):
         db_config=MilvusConfig(
             db_label=parameters["db_label"],
             uri=SecretStr(parameters["uri"]),
+            user=parameters["user_name"],
+            password=SecretStr(parameters["password"]),
         ),
         db_case_config=IVFSQ8Config(
             nlist=parameters["nlist"],
@@ -143,6 +159,8 @@ def MilvusDISKANN(**parameters: Unpack[MilvusDISKANNTypedDict]):
         db_config=MilvusConfig(
             db_label=parameters["db_label"],
             uri=SecretStr(parameters["uri"]),
+            user=parameters["user_name"],
+            password=SecretStr(parameters["password"]),
         ),
         db_case_config=DISKANNConfig(
             search_list=parameters["search_list"],
@@ -174,6 +192,8 @@ def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]):
         db_config=MilvusConfig(
             db_label=parameters["db_label"],
             uri=SecretStr(parameters["uri"]),
+            user=parameters["user_name"],
+            password=SecretStr(parameters["password"]),
         ),
         db_case_config=GPUIVFFlatConfig(
             nlist=parameters["nlist"],
@@ -208,6 +228,8 @@ def MilvusGPUIVFPQ(**parameters: Unpack[MilvusGPUIVFPQTypedDict]):
         db_config=MilvusConfig(
             db_label=parameters["db_label"],
             uri=SecretStr(parameters["uri"]),
+            user=parameters["user_name"],
+            password=SecretStr(parameters["password"]),
         ),
         db_case_config=GPUIVFPQConfig(
             nlist=parameters["nlist"],
@@ -274,6 +296,8 @@ def MilvusGPUCAGRA(**parameters: Unpack[MilvusGPUCAGRATypedDict]):
         db_config=MilvusConfig(
             db_label=parameters["db_label"],
             uri=SecretStr(parameters["uri"]),
+            user=parameters["user_name"],
+            password=SecretStr(parameters["password"]),
         ),
         db_case_config=GPUCAGRAConfig(
             intermediate_graph_degree=parameters["intermediate_graph_degree"],

vectordb_bench/backend/clients/milvus/config.py

@@ -1,12 +1,26 @@
-from pydantic import BaseModel, SecretStr
+from pydantic import BaseModel, SecretStr, validator
 from ..api import DBConfig, DBCaseConfig, MetricType, IndexType
 
 
 class MilvusConfig(DBConfig):
     uri: SecretStr = "http://localhost:19530"
+    user: str | None = None
+    password: SecretStr | None = None
 
     def to_dict(self) -> dict:
-        return {"uri": self.uri.get_secret_value()}
+        return {
+            "uri": self.uri.get_secret_value(),
+            "user": self.user if self.user else None,
+            "password": self.password.get_secret_value() if self.password else None,
+        }
+
+    @validator("*")
+    def not_empty_field(cls, v, field):
+        if field.name in cls.common_short_configs() or field.name in cls.common_long_configs() or field.name in ["user", "password"]:
+            return v
+        if isinstance(v, (str, SecretStr)) and len(v) == 0:
+            raise ValueError("Empty string!")
+        return v
 
 
 class MilvusIndexConfig(BaseModel):

vectordb_bench/backend/clients/milvus/milvus.py

@@ -8,7 +8,7 @@ from typing import Iterable
 from pymilvus import Collection, utility
 from pymilvus import CollectionSchema, DataType, FieldSchema, MilvusException
 
-from ..api import VectorDB, IndexType
+from ..api import VectorDB
 from .config import MilvusIndexConfig
 
 
@@ -66,8 +66,7 @@ class Milvus(VectorDB):
                 self.case_config.index_param(),
                 index_name=self._index_name,
             )
-            if kwargs.get("pre_load") is True:
-                self._pre_load(col)
+            col.load()
 
         connections.disconnect("default")
 
@@ -90,16 +89,15 @@ class Milvus(VectorDB):
         connections.disconnect("default")
 
     def _optimize(self):
-        self._post_insert()
         log.info(f"{self.name} optimizing before search")
+        self._post_insert()
         try:
-            self.col.load()
+            self.col.load(refresh=True)
         except Exception as e:
             log.warning(f"{self.name} optimize error: {e}")
             raise e from None
 
     def _post_insert(self):
-        log.info(f"{self.name} post insert before optimize")
         try:
             self.col.flush()
             # wait for index done and load refresh
@@ -130,7 +128,7 @@ class Milvus(VectorDB):
                     log.warning(f"{self.name} compact error: {e}")
                     if hasattr(e, 'code'):
                         if e.code().name == 'PERMISSION_DENIED':
-                            log.warning(f"Skip compact due to permission denied.")
+                            log.warning("Skip compact due to permission denied.")
                             pass
                     else:
                         raise e

vectordb_bench/backend/runner/rate_runner.py

@@ -1,5 +1,6 @@
 import logging
 import time
+import concurrent
 from concurrent.futures import ThreadPoolExecutor
 import multiprocessing as mp
 
@@ -9,7 +10,7 @@ from vectordb_bench.backend.dataset import DataSetIterator
 from vectordb_bench.backend.utils import time_it
 from vectordb_bench import config
 
-from .util import get_data, is_futures_completed, get_future_exceptions
+from .util import get_data
 log = logging.getLogger(__name__)
 
 
@@ -54,26 +55,42 @@ class RatedMultiThreadingInsertRunner:
                     start_time = time.perf_counter()
                     finished, elapsed_time = submit_by_rate()
                     if finished is True:
-                        q.put(None, block=True)
+                        q.put(True, block=True)
                         log.info(f"End of dataset, left unfinished={len(executing_futures)}")
-                        return
+                        break
 
-                    q.put(True, block=False)
+                    q.put(False, block=False)
                     wait_interval = 1 - elapsed_time if elapsed_time < 1 else 0.001
 
-                    e, completed = is_futures_completed(executing_futures, wait_interval)
-                    if completed is True:
-                        ex = get_future_exceptions(executing_futures)
-                        if ex is not None:
-                            log.warn(f"task error, terminating, err={ex}")
-                            q.put(None)
-                            executor.shutdown(wait=True, cancel_futures=True)
-                            raise ex
+                    try:
+                        done, not_done = concurrent.futures.wait(
+                            executing_futures,
+                            timeout=wait_interval,
+                            return_when=concurrent.futures.FIRST_EXCEPTION)
+
+                        if len(not_done) > 0:
+                            log.warning(f"Failed to finish all tasks in 1s, [{len(not_done)}/{len(executing_futures)}] tasks are not done, waited={wait_interval:.2f}, trying to wait in the next round")
+                            executing_futures = list(not_done)
                         else:
                             log.debug(f"Finished {len(executing_futures)} insert-{config.NUM_PER_BATCH} task in 1s, wait_interval={wait_interval:.2f}")
-                        executing_futures = []
-                    else:
-                        log.warning(f"Failed to finish tasks in 1s, {e}, waited={wait_interval:.2f}, try to check the next round")
+                            executing_futures = []
+                    except Exception as e:
+                            log.warn(f"task error, terminating, err={e}")
+                            q.put(None, block=True)
+                            executor.shutdown(wait=True, cancel_futures=True)
+                            raise e
+
                     dur = time.perf_counter() - start_time
                     if dur < 1:
                         time.sleep(1 - dur)
+
+                # wait for all tasks in executing_futures to complete
+                if len(executing_futures) > 0:
+                    try:
+                        done, _ = concurrent.futures.wait(executing_futures,
+                           return_when=concurrent.futures.FIRST_EXCEPTION)
+                    except Exception as e:
+                        log.warn(f"task error, terminating, err={e}")
+                        q.put(None, block=True)
+                        executor.shutdown(wait=True, cancel_futures=True)
+                        raise e

vectordb_bench/backend/runner/read_write_runner.py

@@ -24,7 +24,7 @@ class ReadWriteRunner(MultiProcessingSearchRunner, RatedMultiThreadingInsertRunn
         k: int = 100,
         filters: dict | None = None,
         concurrencies: Iterable[int] = (1, 15, 50),
-        search_stage: Iterable[float] = (0.5, 0.6, 0.7, 0.8, 0.9, 1.0), # search in any insert portion, 0.0 means search from the start
+        search_stage: Iterable[float] = (0.5, 0.6, 0.7, 0.8, 0.9), # search from insert portion, 0.0 means search from the start
         read_dur_after_write: int = 300, # seconds, search duration when insertion is done
         timeout: float | None = None,
     ):
@@ -32,7 +32,7 @@ class ReadWriteRunner(MultiProcessingSearchRunner, RatedMultiThreadingInsertRunn
         self.data_volume = dataset.data.size
 
         for stage in search_stage:
-            assert 0.0 <= stage <= 1.0, "each search stage should be in [0.0, 1.0]"
+            assert 0.0 <= stage < 1.0, "each search stage should be in [0.0, 1.0)"
         self.search_stage = sorted(search_stage)
         self.read_dur_after_write = read_dur_after_write
 
@@ -65,48 +65,114 @@ class ReadWriteRunner(MultiProcessingSearchRunner, RatedMultiThreadingInsertRunn
             k=k,
         )
 
+    def run_optimize(self):
+        """Optimize needs to run in differenct process for pymilvus schema recursion problem"""
+        with self.db.init():
+            log.info("Search after write - Optimize start")
+            self.db.optimize()
+            log.info("Search after write - Optimize finished")
+
+    def run_search(self):
+        log.info("Search after write - Serial search start")
+        res, ssearch_dur = self.serial_search_runner.run()
+        recall, ndcg, p99_latency = res
+        log.info(f"Search after write - Serial search - recall={recall}, ndcg={ndcg}, p99={p99_latency}, dur={ssearch_dur:.4f}")
+        log.info(f"Search after wirte - Conc search start, dur for each conc={self.read_dur_after_write}")
+        max_qps = self.run_by_dur(self.read_dur_after_write)
+        log.info(f"Search after wirte - Conc search finished, max_qps={max_qps}")
+
+        return (max_qps, recall, ndcg, p99_latency)
+
     def run_read_write(self):
-        futures = []
         with mp.Manager() as m:
             q = m.Queue()
             with concurrent.futures.ProcessPoolExecutor(mp_context=mp.get_context("spawn"), max_workers=2) as executor:
-                futures.append(executor.submit(self.run_with_rate, q))
-                futures.append(executor.submit(self.run_search_by_sig, q))
-
-                for future in concurrent.futures.as_completed(futures):
-                    res = future.result()
-                    log.info(f"Result = {res}")
-
+                read_write_futures = []
+                read_write_futures.append(executor.submit(self.run_with_rate, q))
+                read_write_futures.append(executor.submit(self.run_search_by_sig, q))
+
+                try:
+                    for f in concurrent.futures.as_completed(read_write_futures):
+                        res = f.result()
+                        log.info(f"Result = {res}")
+
+                    # Wait for read_write_futures finishing and do optimize and search
+                    op_future = executor.submit(self.run_optimize)
+                    op_future.result()
+
+                    search_future = executor.submit(self.run_search)
+                    last_res = search_future.result()
+
+                    log.info(f"Max QPS after optimze and search: {last_res}")
+                except Exception as e:
+                    log.warning(f"Read and write error: {e}")
+                    executor.shutdown(wait=True, cancel_futures=True)
+                    raise e
         log.info("Concurrent read write all done")
 
-
     def run_search_by_sig(self, q):
-        res = []
+        """
+        Args:
+            q: multiprocessing queue
+                (None) means abnormal exit
+                (False) means updating progress
+                (True) means normal exit
+        """
+        result, start_batch = [], 0
         total_batch = math.ceil(self.data_volume / self.insert_rate)
-        batch = 0
-        recall = 'x'
+        recall, ndcg, p99_latency = None, None, None
+
+        def wait_next_target(start, target_batch) -> bool:
+            """Return False when receive True or None"""
+            while start < target_batch:
+                sig = q.get(block=True)
+
+                if sig is None or sig is True:
+                    return False
+                else:
+                    start += 1
+            return True
 
         for idx, stage in enumerate(self.search_stage):
             target_batch = int(total_batch * stage)
-            while q.get(block=True):
-                batch += 1
-                if batch >= target_batch:
-                    perc = int(stage * 100)
-                    log.info(f"Insert {perc}% done, total batch={total_batch}")
-                    log.info(f"[{batch}/{total_batch}] Serial search - {perc}% start")
-                    recall, ndcg, p99 =self.serial_search_runner.run()
-
-                    if idx < len(self.search_stage) - 1:
-                        stage_search_dur = (self.data_volume  * (self.search_stage[idx + 1] - stage) // self.insert_rate) // len(self.concurrencies)
-                        if stage_search_dur < 30:
-                            log.warning(f"Search duration too short, please reduce concurrency count or insert rate, or increase dataset volume: dur={stage_search_dur}, concurrencies={len(self.concurrencies)}, insert_rate={self.insert_rate}")
-                        log.info(f"[{batch}/{total_batch}] Conc search - {perc}% start, dur for each conc={stage_search_dur}s")
-                    else:
-                        last_search_dur = self.data_volume * (1.0 - stage) // self.insert_rate
-                        stage_search_dur = last_search_dur + self.read_dur_after_write
-                        log.info(f"[{batch}/{total_batch}] Last conc search - {perc}% start, [read_until_write|read_after_write|total] =[{last_search_dur}s|{self.read_dur_after_write}s|{stage_search_dur}s]")
-
-                    max_qps = self.run_by_dur(stage_search_dur)
-                    res.append((perc, max_qps, recall))
-                    break
-        return res
+            perc = int(stage * 100)
+
+            got = wait_next_target(start_batch, target_batch)
+            if got is False:
+                log.warning(f"Abnormal exit, target_batch={target_batch}, start_batch={start_batch}")
+                return
+
+            log.info(f"Insert {perc}% done, total batch={total_batch}")
+            log.info(f"[{target_batch}/{total_batch}] Serial search - {perc}% start")
+            res, ssearch_dur = self.serial_search_runner.run()
+            recall, ndcg, p99_latency = res
+            log.info(f"[{target_batch}/{total_batch}] Serial search - {perc}% done, recall={recall}, ndcg={ndcg}, p99={p99_latency}, dur={ssearch_dur:.4f}")
+
+            # Search duration for non-last search stage is carefully calculated.
+            # If duration for each concurrency is less than 30s, runner will raise error.
+            if idx < len(self.search_stage) - 1:
+                total_dur_between_stages = self.data_volume  * (self.search_stage[idx + 1] - stage) // self.insert_rate
+                csearch_dur = total_dur_between_stages - ssearch_dur
+
+                # Try to leave room for init process executors
+                csearch_dur = csearch_dur - 30 if csearch_dur > 60 else csearch_dur
+
+                each_conc_search_dur = csearch_dur / len(self.concurrencies)
+                if each_conc_search_dur < 30:
+                    warning_msg = f"Results might be inaccurate, duration[{csearch_dur:.4f}] left for conc-search is too short, total available dur={total_dur_between_stages}, serial_search_cost={ssearch_dur}."
+                    log.warning(warning_msg)
+
+            # The last stage
+            else:
+                each_conc_search_dur = 60
+
+            log.info(f"[{target_batch}/{total_batch}] Concurrent search - {perc}% start, dur={each_conc_search_dur:.4f}")
+            max_qps = self.run_by_dur(each_conc_search_dur)
+            result.append((perc, max_qps, recall, ndcg, p99_latency))
+
+            start_batch = target_batch
+
+        # Drain the queue
+        while q.empty() is False:
+            q.get(block=True)
+        return result

vectordb_bench/backend/runner/serial_runner.py

@@ -167,7 +167,7 @@ class SerialSearchRunner:
             self.test_data = test_data
         self.ground_truth = ground_truth
 
-    def search(self, args: tuple[list, pd.DataFrame]):
+    def search(self, args: tuple[list, pd.DataFrame]) -> tuple[float, float, float]:
         log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency")
         with self.db.init():
             test_data, ground_truth = args
@@ -224,5 +224,11 @@ class SerialSearchRunner:
             result = future.result()
             return result
 
-    def run(self) -> tuple[float, float]:
+    @utils.time_it
+    def run(self) -> tuple[float, float, float]:
+        """
+        Returns:
+            tuple[tuple[float, float, float], float]: (avg_recall, avg_ndcg, p99_latency), cost
+
+        """
         return self._run_in_subprocess()

vectordb_bench/backend/runner/util.py

@@ -1,6 +1,4 @@
 import logging
-import concurrent
-from typing import Iterable
 
 from pandas import DataFrame
 import numpy as np
@@ -16,17 +14,3 @@ def get_data(data_df: DataFrame, normalize: bool) -> tuple[list[list[float]], li
     else:
         all_embeddings = emb_np.tolist()
     return all_embeddings, all_metadata
-
-def is_futures_completed(futures: Iterable[concurrent.futures.Future], interval) -> (Exception, bool):
-    try:
-        list(concurrent.futures.as_completed(futures, timeout=interval))
-    except TimeoutError as e:
-        return e, False
-    return None, True
-
-
-def get_future_exceptions(futures: Iterable[concurrent.futures.Future]) -> BaseException | None:
-    for f in futures:
-        if f.exception() is not None:
-            return f.exception()
-    return

vectordb_bench/backend/task_runner.py

@@ -206,7 +206,7 @@ class CaseRunner(BaseModel):
         finally:
             runner = None
 
-    def _serial_search(self) -> tuple[float, float]:
+    def _serial_search(self) -> tuple[float, float, float]:
         """Performance serial tests, search the entire test data once,
         calculate the recall, serial_latency_p99
 
@@ -214,7 +214,8 @@ class CaseRunner(BaseModel):
             tuple[float, float]: recall, serial_latency_p99
         """
         try:
-            return self.serial_search_runner.run()
+            results, _ = self.serial_search_runner.run()
+            return results
         except Exception as e:
             log.warning(f"search error: {str(e)}, {e}")
             self.stop()
@@ -238,7 +239,7 @@ class CaseRunner(BaseModel):
     @utils.time_it
     def _task(self) -> None:
         with self.db.init():
-            self.db.optimize()
+            self.db.optimize_with_size(data_size=self.ca.dataset.data.size)
 
     def _optimize(self) -> float:
         with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor:

vectordb_bench/backend/utils.py

@@ -35,6 +35,7 @@ def numerize(n) -> str:
 
 
 def time_it(func):
+    """ returns result and elapsed time"""
     @wraps(func)
     def inner(*args, **kwargs):
         pref = time.perf_counter()

vectordb_bench/frontend/config/dbCaseConfigs.py

@@ -437,6 +437,16 @@ CaseConfigParamInput_EF_SEARCH_AWSOpensearch = CaseConfigInput(
     },
 )
 
+CaseConfigParamInput_EF_SEARCH_AliyunOpensearch = CaseConfigInput(
+    label=CaseConfigParamType.ef_search,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 1,
+        "max": 1000000,
+        "value": 40,
+    },
+)
+
 
 CaseConfigParamInput_maintenance_work_mem_PgVector = CaseConfigInput(
     label=CaseConfigParamType.maintenance_work_mem,
@@ -1040,6 +1050,35 @@ CaseConfigParamInput_max_parallel_workers_AlloyDB = CaseConfigInput(
     },
 )
 
+CaseConfigParamInput_EFConstruction_AliES = CaseConfigInput(
+    label=CaseConfigParamType.EFConstruction,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 8,
+        "max": 512,
+        "value": 360,
+    },
+)
+
+CaseConfigParamInput_M_AliES = CaseConfigInput(
+    label=CaseConfigParamType.M,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 4,
+        "max": 64,
+        "value": 30,
+    },
+)
+CaseConfigParamInput_NumCandidates_AliES = CaseConfigInput(
+    label=CaseConfigParamType.numCandidates,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 1,
+        "max": 10000,
+        "value": 100,
+    },
+)
+
 
 MilvusLoadConfig = [
     CaseConfigParamInput_IndexType,
@@ -1099,6 +1138,11 @@ AWSOpenSearchPerformanceConfig = [
     CaseConfigParamInput_EF_SEARCH_AWSOpensearch,
 ]
 
+AliyunOpensearchLoadingConfig = []
+AliyunOpenSearchPerformanceConfig = [
+    CaseConfigParamInput_EF_SEARCH_AliyunOpensearch,
+]
+
 PgVectorLoadingConfig = [
     CaseConfigParamInput_IndexType_PgVector,
     CaseConfigParamInput_Lists_PgVector,
@@ -1206,6 +1250,12 @@ AlloyDBPerformanceConfig = [
     CaseConfigParamInput_max_parallel_workers_AlloyDB,
 ]
 
+AliyunElasticsearchLoadingConfig = [CaseConfigParamInput_EFConstruction_AliES, CaseConfigParamInput_M_AliES]
+AliyunElasticsearchPerformanceConfig = [
+    CaseConfigParamInput_EFConstruction_AliES,
+    CaseConfigParamInput_M_AliES,
+    CaseConfigParamInput_NumCandidates_AliES,
+]
 
 CASE_CONFIG_MAP = {
     DB.Milvus: {
@@ -1247,4 +1297,12 @@ CASE_CONFIG_MAP = {
         CaseLabel.Load: AlloyDBLoadConfig,
         CaseLabel.Performance: AlloyDBPerformanceConfig,
     },
+    DB.AliyunElasticsearch: {
+        CaseLabel.Load: AliyunElasticsearchLoadingConfig,
+        CaseLabel.Performance: AliyunElasticsearchPerformanceConfig,
+    },
+    DB.AliyunOpenSearch: {
+        CaseLabel.Load: AliyunOpensearchLoadingConfig,
+        CaseLabel.Performance: AliyunOpenSearchPerformanceConfig,
+    },
 }

{vectordb_bench-0.0.18.dist-info → vectordb_bench-0.0.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vectordb-bench
-Version: 0.0.18
+Version: 0.0.19
 Summary: VectorDBBench is not just an offering of benchmark results for mainstream vector databases and cloud services, it's your go-to tool for the ultimate performance and cost-effectiveness comparison. Designed with ease-of-use in mind, VectorDBBench is devised to help users, even non-professionals, reproduce results or test new systems, making the hunt for the optimal choice amongst a plethora of cloud services and open-source vector databases a breeze.
 Author-email: XuanYang-cn <xuan.yang@zilliz.com>
 Project-URL: repository, https://github.com/zilliztech/VectorDBBench
@@ -35,15 +35,16 @@ Requires-Dist: qdrant-client; extra == "all"
 Requires-Dist: pinecone-client; extra == "all"
 Requires-Dist: weaviate-client; extra == "all"
 Requires-Dist: elasticsearch; extra == "all"
-Requires-Dist: pgvector; extra == "all"
-Requires-Dist: pgvecto_rs[psycopg3]>=0.2.2; extra == "all"
 Requires-Dist: sqlalchemy; extra == "all"
 Requires-Dist: redis; extra == "all"
 Requires-Dist: chromadb; extra == "all"
+Requires-Dist: pgvector; extra == "all"
 Requires-Dist: psycopg; extra == "all"
 Requires-Dist: psycopg-binary; extra == "all"
-Requires-Dist: opensearch-dsl==2.1.0; extra == "all"
-Requires-Dist: opensearch-py==2.6.0; extra == "all"
+Requires-Dist: pgvecto_rs[psycopg3]>=0.2.2; extra == "all"
+Requires-Dist: opensearch-dsl; extra == "all"
+Requires-Dist: opensearch-py; extra == "all"
+Requires-Dist: memorydb; extra == "all"
 Provides-Extra: qdrant
 Requires-Dist: qdrant-client; extra == "qdrant"
 Provides-Extra: pinecone
@@ -56,18 +57,6 @@ Provides-Extra: pgvector
 Requires-Dist: psycopg; extra == "pgvector"
 Requires-Dist: psycopg-binary; extra == "pgvector"
 Requires-Dist: pgvector; extra == "pgvector"
-Provides-Extra: pgvectorscale
-Requires-Dist: psycopg; extra == "pgvectorscale"
-Requires-Dist: psycopg-binary; extra == "pgvectorscale"
-Requires-Dist: pgvector; extra == "pgvectorscale"
-Provides-Extra: pgdiskann
-Requires-Dist: psycopg; extra == "pgdiskann"
-Requires-Dist: psycopg-binary; extra == "pgdiskann"
-Requires-Dist: pgvector; extra == "pgdiskann"
-Provides-Extra: alloydb
-Requires-Dist: psycopg; extra == "alloydb"
-Requires-Dist: psycopg-binary; extra == "alloydb"
-Requires-Dist: pgvector; extra == "alloydb"
 Provides-Extra: pgvecto-rs
 Requires-Dist: pgvecto_rs[psycopg3]>=0.2.2; extra == "pgvecto-rs"
 Provides-Extra: redis
@@ -76,9 +65,8 @@ Provides-Extra: memorydb
 Requires-Dist: memorydb; extra == "memorydb"
 Provides-Extra: chromadb
 Requires-Dist: chromadb; extra == "chromadb"
-Provides-Extra: awsopensearch
-Requires-Dist: awsopensearch; extra == "awsopensearch"
-Provides-Extra: zilliz-cloud
+Provides-Extra: opensearch
+Requires-Dist: opensearch-py; extra == "opensearch"
 
 # VectorDBBench: A Benchmark Tool for VectorDB
 
@@ -111,21 +99,18 @@ All the database client supported
 
 | Optional database client | install command                             |
 |--------------------------|---------------------------------------------|
-| pymilvus(*default*)      | `pip install vectordb-bench`                |
-| all                      | `pip install vectordb-bench[all]`           |
+| pymilvus, zilliz_cloud (*default*)     | `pip install vectordb-bench`                |
+| all (*clients requirements might be conflict with each other*) | `pip install vectordb-bench[all]`           |
 | qdrant                   | `pip install vectordb-bench[qdrant]`        |
 | pinecone                 | `pip install vectordb-bench[pinecone]`      |
 | weaviate                 | `pip install vectordb-bench[weaviate]`      |
-| elastic                  | `pip install vectordb-bench[elastic]`       |
-| pgvector                 | `pip install vectordb-bench[pgvector]`      |
+| elastic, aliyun_elasticsearch| `pip install vectordb-bench[elastic]`       |
+| pgvector, pgvectorscale, pgdiskann, alloydb | `pip install vectordb-bench[pgvector]`      |
 | pgvecto.rs               | `pip install vectordb-bench[pgvecto_rs]`    |
-| pgvectorscale            | `pip install vectordb-bench[pgvectorscale]` |
-| pgdiskann                | `pip install vectordb-bench[pgdiskann]`     |
 | redis                    | `pip install vectordb-bench[redis]`         |
 | memorydb                 | `pip install vectordb-bench[memorydb]`      |
 | chromadb                 | `pip install vectordb-bench[chromadb]`      |
-| awsopensearch            | `pip install vectordb-bench[awsopensearch]` |
-| alloydb                  | `pip install vectordb-bench[alloydb]`       |
+| awsopensearch            | `pip install vectordb-bench[opensearch]` |
 
 ### Run
 

{vectordb_bench-0.0.18.dist-info → vectordb_bench-0.0.19.dist-info}/RECORD

@@ -7,16 +7,20 @@ vectordb_bench/metric.py,sha256=c-LAxCtb55txnsfd3FN4gRpRif8RREhKRF0eg2JmHGc,2045
 vectordb_bench/models.py,sha256=_0_hscKUqaCHjdjyO_-ntPFgJvgU01y8aldqDcq9ELQ,11041
 vectordb_bench/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vectordb_bench/backend/assembler.py,sha256=mmoLzWXFSlrpWvaVY41wiRNWNv2IR-LzlANX55MJbYI,2028
-vectordb_bench/backend/cases.py,sha256=lQ9jgKaJGunj-mJXR3cgGt16wCsrDrvs-GS3ycTDk0U,16169
+vectordb_bench/backend/cases.py,sha256=tYAXs-8WhkXVkSfUGd5zh51IxKTojBkCgp94eU6Dbwg,16193
 vectordb_bench/backend/data_source.py,sha256=j4-eD0nIe7Y6fSM5WKEij3GfhyU_YOQ3L5Tyl-1GxX0,5446
 vectordb_bench/backend/dataset.py,sha256=MZSu0Q3AkK9gxiuLKNTMH6hhucKK668j4G1-8emhS18,8786
 vectordb_bench/backend/result_collector.py,sha256=jdQf5-q1z5y07SKy9Sig1wFROmm-p9x_Y81fId0sjaU,807
-vectordb_bench/backend/task_runner.py,sha256=cn_RRDyFfNSLlTT84W-ZaXvdl54pK6Cxcsp9ucNRcCs,11864
-vectordb_bench/backend/utils.py,sha256=2UixYyfKvl8zRiashywB1l6hTI3jMtiZhiVm_bXHV1Y,1811
-vectordb_bench/backend/clients/__init__.py,sha256=JKDRKwfDDZLZFct_wahtSEpqu00HVZBNsVyhJ_hHTL8,6701
-vectordb_bench/backend/clients/api.py,sha256=phvqTME3NEPyZGTo85MPeOWwICZO06W3388GT5g72pc,6210
+vectordb_bench/backend/task_runner.py,sha256=ZSWlp5JhVpKTpbKQS3vsq3e5ZtOjz_ET-m4X3Euiicg,11949
+vectordb_bench/backend/utils.py,sha256=8SXcSrw7kmmzeN1cSUwXRSc1BK5dgrj98kALqbrcUNE,1854
+vectordb_bench/backend/clients/__init__.py,sha256=zr3VRNDm7wy8eREMHE4W1LODCZaXhOvt1qCaKQoXyys,7763
+vectordb_bench/backend/clients/api.py,sha256=5OCh_HUWQxE8KCXn9wQMc6jv1TEVnsip02-eUYFoNCo,6285
+vectordb_bench/backend/clients/aliyun_elasticsearch/aliyun_elasticsearch.py,sha256=3bAMrNjhAcWXCSvoW4YT8siY30r4NWqOs9Osv7bgDpo,771
+vectordb_bench/backend/clients/aliyun_elasticsearch/config.py,sha256=FeglLLFvUrjeCOE3vdB5Sg98fufk7lW1NsOHdLTWFsc,564
+vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py,sha256=dWEBroRs-qm17vjUZWIrHOHrLswn6m-NzoMmY1pU9M4,12673
+vectordb_bench/backend/clients/aliyun_opensearch/config.py,sha256=HvBQAENG5Rfiv6VwbvNh9AdyUXm5YTtqa6FdHyPpllc,1311
 vectordb_bench/backend/clients/alloydb/alloydb.py,sha256=rAV558tyd2hX3jcl3bRcxOkeq__GSAXLxfl3MqkAVkM,13375
-vectordb_bench/backend/clients/alloydb/cli.py,sha256=5g3heAEfuwIHCUIHDU4LYQq-CaQto0sGAdr45jdhuNc,4970
+vectordb_bench/backend/clients/alloydb/cli.py,sha256=IoRG0A5O0JIDNpKYoFFAr1czz4QZNihUHx4d1QFA7eQ,4974
 vectordb_bench/backend/clients/alloydb/config.py,sha256=JFQMHvBWG1P5T4N7B95o4tMfN4cVqb01I5TNvjDYQuw,5358
 vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py,sha256=O42OU7K7L0KcJ96AphjmgkyN7a220VLsOL0QwghY2aw,8038
 vectordb_bench/backend/clients/aws_opensearch/cli.py,sha256=v1bGoovgokhIGN5tZwb_MrP4af7BfXYQaOpDuy0Ibh0,1327
@@ -29,9 +33,9 @@ vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py,sha256=rWHthqGEpYw
 vectordb_bench/backend/clients/memorydb/cli.py,sha256=BqU5s1CnLCXeHnSOEpQBON8wWMngeLjvnf9-UQqU9cU,2624
 vectordb_bench/backend/clients/memorydb/config.py,sha256=PjhLMMr_LdJ8O91JpHNCCT6HMEGLwH9r_erUMGJEVaI,1501
 vectordb_bench/backend/clients/memorydb/memorydb.py,sha256=XIqtXpY-2lJohIuImFDsRO3c_upn04eCplIOlaLxFo4,10114
-vectordb_bench/backend/clients/milvus/cli.py,sha256=QqzYIOeUSXEvdLH0_YUMhwDHUDJirTNKeUxrJQIqSdw,8506
-vectordb_bench/backend/clients/milvus/config.py,sha256=AZ4QHoufRIjsX2eVrtnug8SeYnuHeBMna_34OQNFxz0,6847
-vectordb_bench/backend/clients/milvus/milvus.py,sha256=7l2ilpZeCVnXLkWjut2EoIhUraYL7qWfSsBrRaq4vuo,7700
+vectordb_bench/backend/clients/milvus/cli.py,sha256=n3VlULuQQTxDZNN6NJJl3JRzzVfuyJ_AZphl0aoa1Wo,9690
+vectordb_bench/backend/clients/milvus/config.py,sha256=sol9VCAfbrQI6FKSH2RRZh3JFreSzvkGmUVfpSnBSEw,7425
+vectordb_bench/backend/clients/milvus/milvus.py,sha256=fA_A0HkVWZRiGOflmcjw6g1RzF3NDjhfhQc3vBafpEQ,7579
 vectordb_bench/backend/clients/pgdiskann/cli.py,sha256=ued1DyufltataIk6KcmBkNp8PdB9Aj65nVJ6WhrD_VI,3130
 vectordb_bench/backend/clients/pgdiskann/config.py,sha256=8E0GLgUxa5LlJ_eXCugbbO08qdbCVqc1wtdsoOsKEW4,4444
 vectordb_bench/backend/clients/pgdiskann/pgdiskann.py,sha256=bEcbpTVSFxRJ5HiJTX77cgu6NqTMPs8qiGeMF7jBC30,12628
@@ -62,10 +66,10 @@ vectordb_bench/backend/clients/zilliz_cloud/config.py,sha256=3Tk7X4r0n2SLzan110x
 vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py,sha256=4JcwiVEJcdEykW6n471nfHeIlmhIDa-gOZ7G5H_4krY,681
 vectordb_bench/backend/runner/__init__.py,sha256=5dZfPky8pY9Bi9HD5GZ3Fge8V2FJWrkGkQUkNL2v1t0,230
 vectordb_bench/backend/runner/mp_runner.py,sha256=sPJJWg6bKSQYsyWEe5y_j8i_Cf9l5buhtyY-wZxXDAI,9080
-vectordb_bench/backend/runner/rate_runner.py,sha256=qLfirLmS9tR0-3jljaWD_AMw_gt6nwhAVVkxhoo4F4A,3195
-vectordb_bench/backend/runner/read_write_runner.py,sha256=B8PD_gRS5K1nFH5004x6ON1Z8TulK7c4QepW3Glltd8,4732
-vectordb_bench/backend/runner/serial_runner.py,sha256=ku1Dtps9JcmwCwZq7eDw0pcP9IN2Zjjg-1VJumXYJpA,9414
-vectordb_bench/backend/runner/util.py,sha256=pGJn-qXWwGXVlmsMulaqH0zXcasDWjsVwwOJeDFWXhc,1032
+vectordb_bench/backend/runner/rate_runner.py,sha256=UlUOvbvicizNSn7TC0Pf4ni1Z_gaHBuYsu-kiBv4Mec,3963
+vectordb_bench/backend/runner/read_write_runner.py,sha256=HBCzoA2UU8x7LHj7R27EoQgd8RuRIAj5xuAmP8fslkU,7427
+vectordb_bench/backend/runner/serial_runner.py,sha256=7ACSCyCmRRXiD-SIZbS_sl3JhEySdXZ-r7uNFhhji8I,9608
+vectordb_bench/backend/runner/util.py,sha256=C-aELWWsooLG3TlBFd2PGqnQ6Z0_j78_nNk2WCLJyPs,521
 vectordb_bench/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vectordb_bench/cli/cli.py,sha256=Z2-vLwvnnZFsVAPyjFK557cZZYWX_q60XVJP-aYUGdc,15416
 vectordb_bench/cli/vectordbbench.py,sha256=jbpyjh4xKVRocxg4XurLL3ABUzBRXEChRGYhyqH4ItE,1140
@@ -97,7 +101,7 @@ vectordb_bench/frontend/components/run_test/hideSidebar.py,sha256=vb5kzIMmbMqWX6
 vectordb_bench/frontend/components/run_test/initStyle.py,sha256=osPUgfFfH7rRlVNHSMumvmZxvKWlLxmZiNqgnMiUJEU,723
 vectordb_bench/frontend/components/run_test/submitTask.py,sha256=NCEXfR3xudAncjVEvsV2iaiov5AatGObe830UI6481M,3341
 vectordb_bench/frontend/components/tables/data.py,sha256=pVG_hb4bTMLfUt10NUCJSqcFkPmnN7i9jTw9DcWizpI,1364
-vectordb_bench/frontend/config/dbCaseConfigs.py,sha256=40rJ3YtRkPbZBBlWnV7csXDArqrMVaBw6tLMtxzj3w4,36311
+vectordb_bench/frontend/config/dbCaseConfigs.py,sha256=SpTfVtAlLsV9zWrg8zxeGCvLNGlCITWKc7xPN7dPUP8,37921
 vectordb_bench/frontend/config/dbPrices.py,sha256=10aBKjVcEg8y7TPSda28opmBM1KmXNrvbU9WM_BsZcE,176
 vectordb_bench/frontend/config/styles.py,sha256=E2PmwmiewxBKJJ59hQ4ZXatqg8QTN-Z53JlsvWMHM2M,2291
 vectordb_bench/frontend/pages/concurrent.py,sha256=z2izkQ0suO5mZ8PpVY2jypZkF5VT8xUkQQEkwd6C-ww,2094
@@ -123,9 +127,9 @@ vectordb_bench/results/WeaviateCloud/result_20230808_standard_weaviatecloud.json
 vectordb_bench/results/ZillizCloud/result_20230727_standard_zillizcloud.json,sha256=-Mdm4By65XDRCrmVOCF8yQXjcZtH4Xo4shcjoDoBUKU,18293
 vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json,sha256=77XlHT5zM_K7mG5HfDQKwXZnSCuR37VUbt6-P3J_amI,15737
 vectordb_bench/results/ZillizCloud/result_20240105_standard_202401_zillizcloud.json,sha256=TualfJ0664Hs-vdIW68bdkqAEYyzotXmu2P0yIN-GHk,42526
-vectordb_bench-0.0.18.dist-info/LICENSE,sha256=HXbxhrb5u5SegVzeLNF_voVgRsJMavcLaOmD1N0lZkM,1067
-vectordb_bench-0.0.18.dist-info/METADATA,sha256=PtTRr7G1PXhzA88gZYML3Y6JPaqzXk_ejeFbvoRVMOQ,34883
-vectordb_bench-0.0.18.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-vectordb_bench-0.0.18.dist-info/entry_points.txt,sha256=Qzw6gVx96ui8esG21H6yHsI6nboEohRmV424TYhQNrA,113
-vectordb_bench-0.0.18.dist-info/top_level.txt,sha256=jnhZFZAuKX1J60yt-XOeBZ__ctiZMvoC_s0RFq29lpM,15
-vectordb_bench-0.0.18.dist-info/RECORD,,
+vectordb_bench-0.0.19.dist-info/LICENSE,sha256=HXbxhrb5u5SegVzeLNF_voVgRsJMavcLaOmD1N0lZkM,1067
+vectordb_bench-0.0.19.dist-info/METADATA,sha256=Og3-S_xabqf7EyRkgLgr7fE7_nTOyMfwG6BTt-aDJIw,34206
+vectordb_bench-0.0.19.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+vectordb_bench-0.0.19.dist-info/entry_points.txt,sha256=Qzw6gVx96ui8esG21H6yHsI6nboEohRmV424TYhQNrA,113
+vectordb_bench-0.0.19.dist-info/top_level.txt,sha256=jnhZFZAuKX1J60yt-XOeBZ__ctiZMvoC_s0RFq29lpM,15
+vectordb_bench-0.0.19.dist-info/RECORD,,