vectordb-bench 0.0.2__py3-none-any.whl → 0.0.3__py3-none-any.whl

vectordb_bench/backend/runner/serial_runner.py

@@ -4,94 +4,96 @@ import traceback
 import concurrent
 import multiprocessing as mp
 import math
+import psutil
+
 import numpy as np
 import pandas as pd
 
 from ..clients import api
 from ...metric import calc_recall
-from ...models import LoadTimeoutError
+from ...models import LoadTimeoutError, PerformanceTimeoutError
 from .. import utils
 from ... import config
+from vectordb_bench.backend.dataset import DatasetManager
 
 NUM_PER_BATCH = config.NUM_PER_BATCH
-LOAD_TIMEOUT = 24 * 60 * 60
 LOAD_MAX_TRY_COUNT = 10
 WAITTING_TIME = 60
 
 log = logging.getLogger(__name__)
 
-
 class SerialInsertRunner:
-    def __init__(self, db: api.VectorDB, train_emb: list[list[float]], train_id: list[int]):
-        log.debug(f"Dataset shape: {len(train_emb)}")
+    def __init__(self, db: api.VectorDB, dataset: DatasetManager, normalize: bool, timeout: float | None = None):
+        self.timeout = timeout if isinstance(timeout, (int, float)) else None
+        self.dataset = dataset
         self.db = db
-        self.shared_emb = train_emb
-        self.train_id = train_id
-
-        self.seq_batches = math.ceil(len(train_emb)/NUM_PER_BATCH)
+        self.normalize = normalize
 
-    def insert_data(self, left_id: int = 0) -> int:
+    def task(self) -> int:
+        count = 0
         with self.db.init():
-            all_embeddings = self.shared_emb
-
-            # unique id for endlessness insertion
-            all_metadata = [i+left_id for i in self.train_id]
-
-            num_conc_batches = math.ceil(len(all_embeddings)/NUM_PER_BATCH)
-            log.info(f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} embeddings in batch {NUM_PER_BATCH}")
-            count = 0
-            for batch_id in range(self.seq_batches):
-                metadata = all_metadata[batch_id*NUM_PER_BATCH: (batch_id+1)*NUM_PER_BATCH]
-                embeddings = all_embeddings[batch_id*NUM_PER_BATCH: (batch_id+1)*NUM_PER_BATCH]
-
-                log.debug(f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_conc_batches}], Start inserting {len(metadata)} embeddings")
+            log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}")
+            start = time.perf_counter()
+            for data_df in self.dataset:
+                all_metadata = data_df['id'].tolist()
+
+                emb_np = np.stack(data_df['emb'])
+                if self.normalize:
+                    log.debug("normalize the 100k train data")
+                    all_embeddings = emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis].tolist()
+                else:
+                    all_embeddings = emb_np.tolist()
+                del(emb_np)
+                log.debug(f"batch dataset size: {len(all_embeddings)}, {len(all_metadata)}")
+
+                last_batch = self.dataset.data.size - count == len(all_metadata)
                 insert_count, error = self.db.insert_embeddings(
-                    embeddings=embeddings,
-                    metadata=metadata,
+                    embeddings=all_embeddings,
+                    metadata=all_metadata,
+                    last_batch=last_batch,
                 )
-                if error != None:
+                if error is not None:
                     raise error
-                log.debug(f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_conc_batches}], Finish inserting {len(metadata)} embeddings")
 
-                assert insert_count == len(metadata)
+                assert insert_count == len(all_metadata)
                 count += insert_count
-            log.info(f"({mp.current_process().name:16}) Finish inserting {len(all_embeddings)} embeddings in batch {NUM_PER_BATCH}")
-        return count
+                if count % 100_000 == 0:
+                    log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB")
 
-    def endless_insert_data(self, left_id: int = 0) -> int:
-        with self.db.init():
-            all_embeddings = self.shared_emb
+            log.info(f"({mp.current_process().name:16}) Finish loading all dataset into VectorDB, dur={time.perf_counter()-start}")
+            return count
 
+    def endless_insert_data(self, all_embeddings, all_metadata, left_id: int = 0) -> int:
+        with self.db.init():
             # unique id for endlessness insertion
-            all_metadata = [i+left_id for i in self.train_id]
+            all_metadata = [i+left_id for i in all_metadata]
 
-            num_conc_batches = math.ceil(len(all_embeddings)/NUM_PER_BATCH)
+            NUM_BATCHES = math.ceil(len(all_embeddings)/NUM_PER_BATCH)
             log.info(f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} embeddings in batch {NUM_PER_BATCH}")
             count = 0
-            for batch_id in range(self.seq_batches):
+            for batch_id in range(NUM_BATCHES):
                 retry_count = 0
                 already_insert_count = 0
                 metadata = all_metadata[batch_id*NUM_PER_BATCH : (batch_id+1)*NUM_PER_BATCH]
                 embeddings = all_embeddings[batch_id*NUM_PER_BATCH : (batch_id+1)*NUM_PER_BATCH]
 
-                log.debug(f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_conc_batches}], Start inserting {len(metadata)} embeddings")
+                log.debug(f"({mp.current_process().name:16}) batch [{batch_id:3}/{NUM_BATCHES}], Start inserting {len(metadata)} embeddings")
                 while retry_count < LOAD_MAX_TRY_COUNT:
-                    previous_beg, current_beg = 0, 0
                     insert_count, error = self.db.insert_embeddings(
                         embeddings=embeddings[already_insert_count :],
                         metadata=metadata[already_insert_count :],
                     )
                     already_insert_count += insert_count
-                    if error != None:
+                    if error is not None:
                         retry_count += 1
                         time.sleep(WAITTING_TIME)
-                       
+
                         log.info(f"Failed to insert data, try {retry_count} time")
                         if retry_count >= LOAD_MAX_TRY_COUNT:
                             raise error
                     else:
                         break
-                log.debug(f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_conc_batches}], Finish inserting {len(metadata)} embeddings")
+                log.debug(f"({mp.current_process().name:16}) batch [{batch_id:3}/{NUM_BATCHES}], Finish inserting {len(metadata)} embeddings")
 
                 assert already_insert_count == len(metadata)
                 count += already_insert_count
@@ -102,30 +104,46 @@ class SerialInsertRunner:
     def _insert_all_batches(self) -> int:
         """Performance case only"""
         with concurrent.futures.ProcessPoolExecutor(mp_context=mp.get_context('spawn'), max_workers=1) as executor:
-            future = executor.submit(self.insert_data)
-            count = future.result()
-            return count
+            future = executor.submit(self.task)
+            try:
+                count = future.result(timeout=self.timeout)
+            except TimeoutError as e:
+                msg = f"VectorDB load dataset timeout in {self.timeout}"
+                log.warning(msg)
+                for pid, _ in executor._processes.items():
+                    psutil.Process(pid).kill()
+                raise PerformanceTimeoutError(msg) from e
+            except Exception as e:
+                log.warning(f"VectorDB load dataset error: {e}")
+                raise e from e
+            else:
+                return count
 
     def run_endlessness(self) -> int:
         """run forever util DB raises exception or crash"""
+        # datasets for load tests are quite small, can fit into memory
+        # only 1 file
+        data_df = [data_df for data_df in self.dataset][0]
+        all_embeddings, all_metadata = np.stack(data_df["emb"]).tolist(), data_df['id'].tolist()
+
         start_time = time.perf_counter()
         max_load_count, times = 0, 0
         try:
             with self.db.init():
                 self.db.ready_to_load()
-            while time.perf_counter() - start_time < config.CASE_TIMEOUT_IN_SECOND:
-                count = self.endless_insert_data(left_id=max_load_count)
+            while time.perf_counter() - start_time < self.timeout:
+                count = self.endless_insert_data(all_embeddings, all_metadata, left_id=max_load_count)
                 max_load_count += count
                 times += 1
                 log.info(f"Loaded {times} entire dataset, current max load counts={utils.numerize(max_load_count)}, {max_load_count}")
-            raise LoadTimeoutError("capacity case load timeout and stop")
-        except LoadTimeoutError as e:
-            log.info("load timetout, stop the load case")
-            raise e from None
         except Exception as e:
             log.info(f"Capacity case load reach limit, insertion counts={utils.numerize(max_load_count)}, {max_load_count}, err={e}")
             traceback.print_exc()
             return max_load_count
+        else:
+            msg = f"capacity case load timeout in {self.timeout}s"
+            log.info(msg)
+            raise LoadTimeoutError(msg)
 
     def run(self) -> int:
         count, dur = self._insert_all_batches()
@@ -157,7 +175,7 @@ class SerialSearchRunner:
             test_data, ground_truth = args
 
             log.debug(f"test dataset size: {len(test_data)}")
-            log.info(f"ground truth size: {ground_truth.columns}, shape: {ground_truth.shape}")
+            log.debug(f"ground truth size: {ground_truth.columns}, shape: {ground_truth.shape}")
 
             latencies, recalls = [], []
             for idx, emb in enumerate(test_data):

vectordb_bench/backend/task_runner.py

@@ -1,4 +1,5 @@
 import logging
+import psutil
 import traceback
 import concurrent
 import numpy as np
@@ -7,7 +8,7 @@ from enum import Enum, auto
 from . import utils
 from .cases import Case, CaseLabel
 from ..base import BaseModel
-from ..models import TaskConfig
+from ..models import TaskConfig, PerformanceTimeoutError
 
 from .clients import (
     api,
@@ -92,80 +93,70 @@ class CaseRunner(BaseModel):
         self._pre_run(drop_old)
 
         if self.ca.label == CaseLabel.Load:
-            return self._run_load_case()
+            return self._run_capacity_case()
         elif self.ca.label == CaseLabel.Performance:
             return self._run_perf_case(drop_old)
         else:
-            log.warning(f"unknown case type: {self.ca.label}")
-            raise ValueError(f"Unknown case type: {self.ca.label}")
+            msg = f"unknown case type: {self.ca.label}"
+            log.warning(msg)
+            raise ValueError(msg)
 
-
-    def _run_load_case(self) -> Metric:
-        """ run load cases
+    def _run_capacity_case(self) -> Metric:
+        """ run capacity cases
 
         Returns:
             Metric: the max load count
         """
         log.info("Start capacity case")
-        # datasets for load tests are quite small, can fit into memory
-        # only 1 file
-        data_df = [data_df for data_df in self.ca.dataset][0]
-
-        all_embeddings, all_metadata = np.stack(data_df["emb"]).tolist(), data_df['id'].tolist()
-        runner = SerialInsertRunner(self.db, all_embeddings, all_metadata)
         try:
+            runner = SerialInsertRunner(self.db, self.ca.dataset, self.normalize, self.ca.load_timeout)
             count = runner.run_endlessness()
-            log.info(f"load reach limit: insertion counts={count}")
-            return Metric(max_load_count=count)
         except Exception as e:
-            log.warning(f"run capacity case error: {e}")
+            log.warning(f"Failed to run capacity case, reason = {e}")
             raise e from None
-        log.info("End capacity case")
-
+        else:
+            log.info(f"Capacity case loading dataset reaches VectorDB's limit: max capacity = {count}")
+            return Metric(max_load_count=count)
 
     def _run_perf_case(self, drop_old: bool = True) -> Metric:
+        """ run performance cases
+
+        Returns:
+            Metric: load_duration, recall, serial_latency_p99, and, qps
+        """
         try:
             m = Metric()
             if drop_old:
                 _, load_dur = self._load_train_data()
                 build_dur = self._optimize()
                 m.load_duration = round(load_dur+build_dur, 4)
+                log.info(
+                    f"Finish loading the entire dataset into VectorDB,"
+                    f" insert_duration={load_dur}, optimize_duration={build_dur}"
+                    f" load_duration(insert + optimize) = {m.load_duration}"
+                )
 
             self._init_search_runner()
             m.recall, m.serial_latency_p99 = self._serial_search()
             m.qps = self._conc_search()
-
-            log.info(f"got results: {m}")
-            return m
         except Exception as e:
-            log.warning(f"performance case run error: {e}")
+            log.warning(f"Failed to run performance case, reason = {e}")
             traceback.print_exc()
-            raise e
+            raise e from None
+        else:
+            log.info(f"Performance case got result: {m}")
+            return m
 
     @utils.time_it
     def _load_train_data(self):
         """Insert train data and get the insert_duration"""
-        for data_df in self.ca.dataset:
-            try:
-                all_metadata = data_df['id'].tolist()
-
-                emb_np = np.stack(data_df['emb'])
-                if self.normalize:
-                    log.debug("normalize the 100k train data")
-                    all_embeddings = emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis].tolist()
-                else:
-                    all_embeddings = emb_np.tolist()
-
-                del(emb_np)
-                log.debug(f"normalized size: {len(all_embeddings)}, {len(all_metadata)}")
-
-                runner = SerialInsertRunner(self.db, all_embeddings, all_metadata)
-                runner.run()
-            except Exception as e:
-                raise e from None
-            finally:
-                runner = None
-
+        try:
+            runner = SerialInsertRunner(self.db, self.ca.dataset, self.normalize, self.ca.load_timeout)
+            runner.run()
+        except Exception as e:
+            raise e from None
+        finally:
+            runner = None
 
     def _serial_search(self) -> tuple[float, float]:
         """Performance serial tests, search the entire test data once,
@@ -198,17 +189,21 @@ class CaseRunner(BaseModel):
 
     @utils.time_it
     def _task(self) -> None:
-        """"""
         with self.db.init():
-            self.db.ready_to_search()
+            self.db.optimize()
 
     def _optimize(self) -> float:
         with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor:
             future = executor.submit(self._task)
             try:
-                return future.result()[1]
+                return future.result(timeout=self.ca.optimize_timeout)[1]
+            except TimeoutError as e:
+                log.warning(f"VectorDB optimize timeout in {self.ca.optimize_timeout}")
+                for pid, _ in executor._processes.items():
+                    psutil.Process(pid).kill()
+                raise PerformanceTimeoutError("Performance case optimize timeout") from e
             except Exception as e:
-                log.warning(f"VectorDB ready_to_search error: {e}")
+                log.warning(f"VectorDB optimize error: {e}")
                 raise e from None
 
     def _init_search_runner(self):

vectordb_bench/frontend/components/get_results/saveAsImage.py

@@ -32,7 +32,8 @@ const stApp = streamlitDoc.querySelector('.main > .block-container');
 const buttons = Array.from(streamlitDoc.querySelectorAll('.stButton > button'));
 const imgButton = buttons.find(el => el.innerText === '{buttonText}');
 
-imgButton.innerText = 'Creating Image...';
+if (imgButton)
+    imgButton.innerText = 'Creating Image...';
 
 html2canvas(stApp, {{ allowTaint: false, useCORS: true }}).then(function (canvas) {{
     a = document.createElement('a');
@@ -40,7 +41,8 @@ html2canvas(stApp, {{ allowTaint: false, useCORS: true }}).then(function (canvas
     a.download = '{pageName}.png';
     a.click();
     
-    imgButton.innerText = '{buttonText}';
+    if (imgButton)
+        imgButton.innerText = '{buttonText}';
 }})
 </script>""",
             height=0,

vectordb_bench/frontend/const/dbCaseConfigs.py

@@ -29,6 +29,7 @@ CASE_LIST_WITH_DIVIDER = [
 
 CASE_LIST = [item for item in CASE_LIST_WITH_DIVIDER if isinstance(item, CaseType)]
 
+
 class InputType(IntEnum):
     Text = 20001
     Number = 20002
@@ -185,6 +186,26 @@ CaseConfigParamInput_Nprobe = CaseConfigInput(
     == IndexType.IVFFlat.value,
 )
 
+CaseConfigParamInput_Lists = CaseConfigInput(
+    label=CaseConfigParamType.lists,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 1,
+        "max": 65536,
+        "value": 10,
+    },
+)
+
+CaseConfigParamInput_Probes = CaseConfigInput(
+    label=CaseConfigParamType.probes,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 1,
+        "max": 65536,
+        "value": 1,
+    },
+)
+
 
 MilvusLoadConfig = [
     CaseConfigParamInput_IndexType,
@@ -192,8 +213,6 @@ MilvusLoadConfig = [
     CaseConfigParamInput_EFConstruction_Milvus,
     CaseConfigParamInput_Nlist,
 ]
-
-
 MilvusPerformanceConfig = [
     CaseConfigParamInput_IndexType,
     CaseConfigParamInput_M,
@@ -208,7 +227,6 @@ WeaviateLoadConfig = [
     CaseConfigParamInput_MaxConnections,
     CaseConfigParamInput_EFConstruction_Weaviate,
 ]
-
 WeaviatePerformanceConfig = [
     CaseConfigParamInput_MaxConnections,
     CaseConfigParamInput_EFConstruction_Weaviate,
@@ -216,13 +234,15 @@ WeaviatePerformanceConfig = [
 ]
 
 ESLoadingConfig = [CaseConfigParamInput_EFConstruction_ES, CaseConfigParamInput_M_ES]
-
 ESPerformanceConfig = [
     CaseConfigParamInput_EFConstruction_ES,
     CaseConfigParamInput_M_ES,
     CaseConfigParamInput_NumCandidates_ES,
 ]
 
+PgVectorLoadingConfig = [CaseConfigParamInput_Lists]
+PgVectorPerformanceConfig = [CaseConfigParamInput_Lists, CaseConfigParamInput_Probes]
+
 CASE_CONFIG_MAP = {
     DB.Milvus: {
         CaseType.CapacityDim960: MilvusLoadConfig,
@@ -257,4 +277,15 @@ CASE_CONFIG_MAP = {
         CaseType.Performance10M99P: ESPerformanceConfig,
         CaseType.Performance1M99P: ESPerformanceConfig,
     },
+    DB.PgVector: {
+        CaseType.CapacityDim960: PgVectorLoadingConfig,
+        CaseType.CapacityDim128: PgVectorLoadingConfig,
+        CaseType.Performance100M: PgVectorPerformanceConfig,
+        CaseType.Performance10M: PgVectorPerformanceConfig,
+        CaseType.Performance1M: PgVectorPerformanceConfig,
+        CaseType.Performance10M1P: PgVectorPerformanceConfig,
+        CaseType.Performance1M1P: PgVectorPerformanceConfig,
+        CaseType.Performance10M99P: PgVectorPerformanceConfig,
+        CaseType.Performance1M99P: PgVectorPerformanceConfig,
+    },
 }

vectordb_bench/frontend/const/dbPrices.py

@@ -1,34 +1,6 @@
-from vectordb_bench.backend.clients import DB
+from vectordb_bench import config
+import ujson
+import pathlib
 
-
-DB_DBLABEL_TO_PRICE = {
-    DB.Milvus.value: {},
-    DB.ZillizCloud.value: {
-        "1cu-perf": 0.159,
-        "8cu-perf": 1.272,
-        "1cu-cap": 0.159,
-        "2cu-cap": 0.318,
-    },
-    DB.WeaviateCloud.value: {
-        # "sandox": 0,
-        "standard": 10.10,
-        "bus_crit": 32.60,
-    },
-    DB.ElasticCloud.value: {
-        "upTo2.5c8g": 0.4793,
-    },
-    DB.QdrantCloud.value: {
-        "0.5c4g-1node": 0.052,
-        "2c8g-1node": 0.166,
-        "4c16g-5node": 1.426,
-    },
-    DB.Pinecone.value: {
-        "s1.x1": 0.0973,
-        "s1.x2": 0.194,
-        "p1.x1": 0.0973,
-        "p2.x1": 0.146,
-        "p2.x1-8node": 1.168,
-        "p1.x1-8node": 0.779,
-        "s1.x1-2node": 0.195,
-    },
-}
+with open(pathlib.Path(config.RESULTS_LOCAL_DIR, "dbPrices.json")) as f:
+    DB_DBLABEL_TO_PRICE = ujson.load(f)

vectordb_bench/frontend/const/styles.py

@@ -32,21 +32,27 @@ PAGE_TITLE = "VectorDB Benchmark"
 FAVICON = "https://assets.zilliz.com/favicon_f7f922fe27.png"
 HEADER_ICON = "https://assets.zilliz.com/vdb_benchmark_db790b5387.png"
 
+# RedisCloud icon: https://assets.zilliz.com/Redis_Cloud_74b8bfef39.png
+# Elasticsearch icon: https://assets.zilliz.com/elasticsearch_beffeadc29.png
+# Chroma icon: https://assets.zilliz.com/chroma_ceb3f06ed7.png
 DB_TO_ICON = {
     DB.Milvus: "https://assets.zilliz.com/milvus_c30b0d1994.png",
     DB.ZillizCloud: "https://assets.zilliz.com/zilliz_5f4cc9b050.png",
-    DB.ElasticCloud: "https://assets.zilliz.com/elasticsearch_beffeadc29.png",
+    DB.ElasticCloud: "https://assets.zilliz.com/Elatic_Cloud_dad8d6a3a3.png",
     DB.Pinecone: "https://assets.zilliz.com/pinecone_94d8154979.png",
     DB.QdrantCloud: "https://assets.zilliz.com/qdrant_b691674fcd.png",
     DB.WeaviateCloud: "https://assets.zilliz.com/weaviate_4f6f171ebe.png",
+    DB.PgVector: "https://assets.zilliz.com/PG_Vector_d464f2ef5f.png",
 }
 
-
+# RedisCloud color: #0D6EFD
+# Chroma color: #FFC107
 COLOR_MAP = {
     DB.Milvus.value: "#0DCAF0",
     DB.ZillizCloud.value: "#0D6EFD",
-    DB.ElasticCloud.value: "#fdc613",
+    DB.ElasticCloud.value: "#04D6C8",
     DB.Pinecone.value: "#6610F2",
     DB.QdrantCloud.value: "#D91AD9",
     DB.WeaviateCloud.value: "#20C997",
+    DB.PgVector.value: "#4C779A",
 }

vectordb_bench/metric.py

@@ -19,7 +19,6 @@ class Metric:
     qps: float = 0.0
     serial_latency_p99: float = 0.0
     recall: float = 0.0
-    quries_per_dollar: float = 0.0
 
 
 QURIES_PER_DOLLAR_METRIC = "QP$ (Quries per Dollar)"

vectordb_bench/models.py

@@ -43,6 +43,8 @@ class CaseConfigParamType(Enum):
     Nprobe = "nprobe"
     MaxConnections = "maxConnections"
     numCandidates = "num_candidates"
+    lists = "lists"
+    probes = "probes"
 
 
 class CustomizedCase(BaseModel):
@@ -167,20 +169,21 @@ class TestResult(BaseModel):
         max_qps = max(map(len, [str(f.metrics.qps) for f in filtered_results])) + 3
         max_recall = max(map(len, [str(f.metrics.recall) for f in filtered_results])) + 3
 
-        max_db_labels = 8 if max_db_labels == 0 else max_db_labels
-        max_load_dur = 11 if max_load_dur == 0 else max_load_dur + 3
-        max_qps = 10 if max_qps == 0 else max_load_dur + 3
-        max_recall = 13 if max_recall == 0 else max_recall + 3
+        max_db_labels = 8 if max_db_labels < 8 else max_db_labels
+        max_load_dur = 11 if max_load_dur < 11 else max_load_dur
+        max_qps = 10 if max_qps < 10 else max_qps
+        max_recall = 13 if max_recall < 13 else max_recall
 
-        LENGTH = (max_db, max_db_labels, max_case, len(self.task_label), max_load_dur, max_qps, 15, max_recall, 14)
+        LENGTH = (max_db, max_db_labels, max_case, len(self.task_label), max_load_dur, max_qps, 15, max_recall, 14, 5)
 
         DATA_FORMAT = (
-            f"%-{max_db}s | %-{max_db_labels}s %-{max_case}s %-{len(self.task_label)}s "
-            f"| %-{max_load_dur}s %-{max_qps}s %-15s %-{max_recall}s %-14s"
+            f"%-{max_db}s | %-{max_db_labels}s %-{max_case}s %-{len(self.task_label)}s"
+            f" | %-{max_load_dur}s %-{max_qps}s %-15s %-{max_recall}s %-14s"
+            f" | %-5s"
         )
 
         TITLE = DATA_FORMAT % (
-            "DB", "db_label", "case", "label", "load_dur", "qps", "latency(p99)", "recall", "max_load_count")
+            "DB", "db_label", "case", "label", "load_dur", "qps", "latency(p99)", "recall", "max_load_count", "label")
         SPLIT = DATA_FORMAT%tuple(map(lambda x:"-"*x, LENGTH))
         SUMMERY_FORMAT = ("Task summery: run_id=%s, task_label=%s") % (self.run_id[:5], self.task_label)
         fmt = [SUMMERY_FORMAT, TITLE, SPLIT]
@@ -197,6 +200,7 @@ class TestResult(BaseModel):
                 f.metrics.serial_latency_p99,
                 f.metrics.recall,
                 f.metrics.max_load_count,
+                f.label.value,
             ))
 
         tmp_logger = logging.getLogger("no_color")

vectordb_bench/results/dbPrices.json

@@ -0,0 +1,32 @@
+{
+  "Milvus": {},
+  "ZillizCloud": {
+    "1cu-perf": 0.159,
+    "8cu-perf": 1.272,
+    "1cu-cap": 0.159,
+    "2cu-cap": 0.318
+  },
+  "WeaviateCloud": {
+    "standard": 10.1,
+    "bus_crit": 32.6
+  },
+  "ElasticCloud": {
+    "upTo2.5c8g": 0.4793
+  },
+  "QdrantCloud": {
+    "0.5c4g-1node": 0.052,
+    "2c8g-1node": 0.166,
+    "4c16g-1node": 0.2852,
+    "4c16g-5node": 1.426
+  },
+  "Pinecone": {
+    "s1.x1": 0.0973,
+    "s1.x2": 0.194,
+    "p1.x1": 0.0973,
+    "p2.x1": 0.146,
+    "p2.x1-8node": 1.168,
+    "p1.x1-8node": 0.779,
+    "s1.x1-2node": 0.195
+  },
+  "PgVector": {}
+}