vectordb-bench 0.0.12__py3-none-any.whl → 0.0.14__py3-none-any.whl

vectordb_bench/frontend/config/dbCaseConfigs.py

@@ -148,6 +148,7 @@ class InputType(IntEnum):
     Text = 20001
     Number = 20002
     Option = 20003
+    Float = 20004
 
 
 class CaseConfigInput(BaseModel):
@@ -169,6 +170,7 @@ CaseConfigParamInput_IndexType = CaseConfigInput(
             IndexType.IVFFlat.value,
             IndexType.IVFSQ8.value,
             IndexType.DISKANN.value,
+            IndexType.STREAMING_DISKANN.value,
             IndexType.Flat.value,
             IndexType.AUTOINDEX.value,
             IndexType.GPU_IVF_FLAT.value,
@@ -178,6 +180,104 @@ CaseConfigParamInput_IndexType = CaseConfigInput(
     },
 )
 
+
+CaseConfigParamInput_IndexType_PgVectorScale = CaseConfigInput(
+    label=CaseConfigParamType.IndexType,
+    inputHelp="Select Index Type",
+    inputType=InputType.Option,
+    inputConfig={
+        "options": [
+            IndexType.STREAMING_DISKANN.value,
+        ],
+    },
+)
+
+
+CaseConfigParamInput_storage_layout = CaseConfigInput(
+    label=CaseConfigParamType.storage_layout,
+    inputHelp="Select Storage Layout",
+    inputType=InputType.Option,
+    inputConfig={
+        "options": [
+            "memory_optimized",
+            "plain",
+        ],
+    },
+)
+
+CaseConfigParamInput_num_neighbors = CaseConfigInput(
+    label=CaseConfigParamType.num_neighbors,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 10,
+        "max": 300,
+        "value": 50,
+    },
+    isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
+    == IndexType.STREAMING_DISKANN.value,
+)
+
+CaseConfigParamInput_search_list_size = CaseConfigInput(
+    label=CaseConfigParamType.search_list_size,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 10,
+        "max": 300,
+        "value": 100,
+    },
+    isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
+    == IndexType.STREAMING_DISKANN.value,
+)
+
+CaseConfigParamInput_max_alpha = CaseConfigInput(
+    label=CaseConfigParamType.max_alpha,
+    inputType=InputType.Float,
+    inputConfig={
+        "min": 0.1,
+        "max": 2.0,
+        "value": 1.2,
+    },
+    isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
+    == IndexType.STREAMING_DISKANN.value,
+)
+
+CaseConfigParamInput_num_dimensions = CaseConfigInput(
+    label=CaseConfigParamType.num_dimensions,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 0,
+        "max": 2000,
+        "value": 0,
+    },
+    isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
+    == IndexType.STREAMING_DISKANN.value,
+)
+
+CaseConfigParamInput_query_search_list_size = CaseConfigInput(
+    label=CaseConfigParamType.query_search_list_size,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 50,
+        "max": 150,
+        "value": 100,
+    },
+    isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
+    == IndexType.STREAMING_DISKANN.value,
+)
+
+
+CaseConfigParamInput_query_rescore = CaseConfigInput(
+    label=CaseConfigParamType.query_rescore,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 0,
+        "max": 150,
+        "value": 50,
+    },
+    isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
+    == IndexType.STREAMING_DISKANN.value,
+)
+
 CaseConfigParamInput_IndexType_PgVector = CaseConfigInput(
     label=CaseConfigParamType.IndexType,
     inputHelp="Select Index Type",
@@ -190,6 +290,19 @@ CaseConfigParamInput_IndexType_PgVector = CaseConfigInput(
     },
 )
 
+CaseConfigParamInput_IndexType_PgVectoRS = CaseConfigInput(
+    label=CaseConfigParamType.IndexType,
+    inputHelp="Select Index Type",
+    inputType=InputType.Option,
+    inputConfig={
+        "options": [
+            IndexType.HNSW.value,
+            IndexType.IVFFlat.value,
+            IndexType.Flat.value,
+        ],
+    },
+)
+
 CaseConfigParamInput_M = CaseConfigInput(
     label=CaseConfigParamType.M,
     inputType=InputType.Number,
@@ -247,6 +360,37 @@ CaseConfigParamInput_EFConstruction_ES = CaseConfigInput(
     },
 )
 
+CaseConfigParamInput_EFConstruction_AWSOpensearch = CaseConfigInput(
+    label=CaseConfigParamType.EFConstruction,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 100,
+        "max": 1024,
+        "value": 256,
+    },
+)
+
+CaseConfigParamInput_M_AWSOpensearch = CaseConfigInput(
+    label=CaseConfigParamType.M,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 4,
+        "max": 64,
+        "value": 16,
+    },
+)
+
+CaseConfigParamInput_EF_SEARCH_AWSOpensearch = CaseConfigInput(
+    label=CaseConfigParamType.ef_search,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 100,
+        "max": 1024,
+        "value": 256,
+    },
+)
+
+
 CaseConfigParamInput_maintenance_work_mem_PgVector = CaseConfigInput(
     label=CaseConfigParamType.maintenance_work_mem,
     inputHelp="Recommended value: 1.33x the index size, not to exceed the available free memory."
@@ -272,14 +416,26 @@ CaseConfigParamInput_max_parallel_workers_PgVector = CaseConfigInput(
 
 
 CaseConfigParamInput_EFConstruction_PgVectoRS = CaseConfigInput(
-    label=CaseConfigParamType.EFConstruction,
+    label=CaseConfigParamType.ef_construction,
     inputType=InputType.Number,
     inputConfig={
-        "min": 8,
-        "max": 512,
-        "value": 360,
+        "min": 10,
+        "max": 2000,
+        "value": 300,
     },
-    isDisplayed=lambda config: config[CaseConfigParamType.IndexType]
+    isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
+    == IndexType.HNSW.value,
+)
+
+CaseConfigParamInput_EFSearch_PgVectoRS = CaseConfigInput(
+    label=CaseConfigParamType.ef_search,
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 1,
+        "max": 65535,
+        "value": 100,
+    },
+    isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
     == IndexType.HNSW.value,
 )
 
@@ -402,6 +558,7 @@ CaseConfigParamInput_M_PQ = CaseConfigInput(
     in [IndexType.GPU_IVF_PQ.value],
 )
 
+
 CaseConfigParamInput_Nbits_PQ = CaseConfigInput(
     label=CaseConfigParamType.nbits,
     inputType=InputType.Number,
@@ -598,6 +755,7 @@ CaseConfigParamInput_EFSearch_PgVector = CaseConfigInput(
     == IndexType.HNSW.value,
 )
 
+
 CaseConfigParamInput_QuantizationType_PgVectoRS = CaseConfigInput(
     label=CaseConfigParamType.quantizationType,
     inputType=InputType.Option,
@@ -611,6 +769,19 @@ CaseConfigParamInput_QuantizationType_PgVectoRS = CaseConfigInput(
     ],
 )
 
+CaseConfigParamInput_QuantizationType_PgVector = CaseConfigInput(
+    label=CaseConfigParamType.quantizationType,
+    inputType=InputType.Option,
+    inputConfig={
+        "options": ["none", "halfvec"],
+    },
+    isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None)
+    in [
+        IndexType.HNSW.value,
+        IndexType.IVFFlat.value,
+    ],
+)
+
 CaseConfigParamInput_QuantizationRatio_PgVectoRS = CaseConfigInput(
     label=CaseConfigParamType.quantizationRatio,
     inputType=InputType.Option,
@@ -626,6 +797,18 @@ CaseConfigParamInput_QuantizationRatio_PgVectoRS = CaseConfigInput(
     ],
 )
 
+CaseConfigParamInput_max_parallel_workers_PgVectorRS = CaseConfigInput(
+    label=CaseConfigParamType.max_parallel_workers,
+    displayLabel="Max parallel workers",
+    inputHelp="Recommended value: (cpu cores - 1). This will set the parameters: [optimizing.optimizing_threads]",
+    inputType=InputType.Number,
+    inputConfig={
+        "min": 0,
+        "max": 1024,
+        "value": 16,
+    },
+)
+
 CaseConfigParamInput_ZillizLevel = CaseConfigInput(
     label=CaseConfigParamType.level,
     inputType=InputType.Number,
@@ -687,11 +870,19 @@ ESPerformanceConfig = [
     CaseConfigParamInput_NumCandidates_ES,
 ]
 
+AWSOpensearchLoadingConfig = [CaseConfigParamInput_EFConstruction_AWSOpensearch, CaseConfigParamInput_M_AWSOpensearch]
+AWSOpenSearchPerformanceConfig = [
+    CaseConfigParamInput_EFConstruction_AWSOpensearch,
+    CaseConfigParamInput_M_AWSOpensearch,
+    CaseConfigParamInput_EF_SEARCH_AWSOpensearch,
+]
+
 PgVectorLoadingConfig = [
     CaseConfigParamInput_IndexType_PgVector,
     CaseConfigParamInput_Lists_PgVector,
     CaseConfigParamInput_m,
     CaseConfigParamInput_EFConstruction_PgVector,
+    CaseConfigParamInput_QuantizationType_PgVector,
     CaseConfigParamInput_maintenance_work_mem_PgVector,
     CaseConfigParamInput_max_parallel_workers_PgVector,
 ]
@@ -702,33 +893,55 @@ PgVectorPerformanceConfig = [
     CaseConfigParamInput_EFSearch_PgVector,
     CaseConfigParamInput_Lists_PgVector,
     CaseConfigParamInput_Probes_PgVector,
+    CaseConfigParamInput_QuantizationType_PgVector,
     CaseConfigParamInput_maintenance_work_mem_PgVector,
     CaseConfigParamInput_max_parallel_workers_PgVector,
 ]
 
 PgVectoRSLoadingConfig = [
-    CaseConfigParamInput_IndexType,
-    CaseConfigParamInput_M,
+    CaseConfigParamInput_IndexType_PgVectoRS,
+    CaseConfigParamInput_m,
     CaseConfigParamInput_EFConstruction_PgVectoRS,
     CaseConfigParamInput_Nlist,
     CaseConfigParamInput_QuantizationType_PgVectoRS,
     CaseConfigParamInput_QuantizationRatio_PgVectoRS,
+    CaseConfigParamInput_max_parallel_workers_PgVectorRS,
 ]
 
 PgVectoRSPerformanceConfig = [
-    CaseConfigParamInput_IndexType,
-    CaseConfigParamInput_M,
+    CaseConfigParamInput_IndexType_PgVectoRS,
+    CaseConfigParamInput_m,
     CaseConfigParamInput_EFConstruction_PgVectoRS,
+    CaseConfigParamInput_EFSearch_PgVectoRS,
     CaseConfigParamInput_Nlist,
     CaseConfigParamInput_Nprobe,
     CaseConfigParamInput_QuantizationType_PgVectoRS,
     CaseConfigParamInput_QuantizationRatio_PgVectoRS,
+    CaseConfigParamInput_max_parallel_workers_PgVectorRS,
 ]
 
 ZillizCloudPerformanceConfig = [
     CaseConfigParamInput_ZillizLevel,
 ]
 
+PgVectorScaleLoadingConfig = [
+    CaseConfigParamInput_IndexType_PgVectorScale,
+    CaseConfigParamInput_num_neighbors,
+    CaseConfigParamInput_storage_layout,
+    CaseConfigParamInput_search_list_size,
+    CaseConfigParamInput_max_alpha,
+]
+
+PgVectorScalePerformanceConfig = [
+    CaseConfigParamInput_IndexType_PgVectorScale,
+    CaseConfigParamInput_num_neighbors,
+    CaseConfigParamInput_storage_layout,
+    CaseConfigParamInput_search_list_size,
+    CaseConfigParamInput_max_alpha,
+    CaseConfigParamInput_query_rescore,
+    CaseConfigParamInput_query_search_list_size,
+]
+
 CASE_CONFIG_MAP = {
     DB.Milvus: {
         CaseLabel.Load: MilvusLoadConfig,
@@ -745,6 +958,10 @@ CASE_CONFIG_MAP = {
         CaseLabel.Load: ESLoadingConfig,
         CaseLabel.Performance: ESPerformanceConfig,
     },
+    DB.AWSOpenSearch: {
+        CaseLabel.Load: AWSOpensearchLoadingConfig,
+        CaseLabel.Performance: AWSOpenSearchPerformanceConfig,
+    },
     DB.PgVector: {
         CaseLabel.Load: PgVectorLoadingConfig,
         CaseLabel.Performance: PgVectorPerformanceConfig,
@@ -753,4 +970,8 @@ CASE_CONFIG_MAP = {
         CaseLabel.Load: PgVectoRSLoadingConfig,
         CaseLabel.Performance: PgVectoRSPerformanceConfig,
     },
+    DB.PgVectorScale: {
+        CaseLabel.Load: PgVectorScaleLoadingConfig,
+        CaseLabel.Performance: PgVectorScalePerformanceConfig,
+    },
 }

vectordb_bench/frontend/pages/quries_per_dollar.py

@@ -1,10 +1,17 @@
 import streamlit as st
 from vectordb_bench.frontend.components.check_results.footer import footer
-from vectordb_bench.frontend.components.check_results.expanderStyle import initMainExpanderStyle
+from vectordb_bench.frontend.components.check_results.expanderStyle import (
+    initMainExpanderStyle,
+)
 from vectordb_bench.frontend.components.check_results.priceTable import priceTable
-from vectordb_bench.frontend.components.check_results.stPageConfig import initResultsPageConfig
+from vectordb_bench.frontend.components.check_results.stPageConfig import (
+    initResultsPageConfig,
+)
 from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
-from vectordb_bench.frontend.components.check_results.nav import NavToResults, NavToRunTest
+from vectordb_bench.frontend.components.check_results.nav import (
+    NavToResults,
+    NavToRunTest,
+)
 from vectordb_bench.frontend.components.check_results.charts import drawMetricChart
 from vectordb_bench.frontend.components.check_results.filters import getshownData
 from vectordb_bench.frontend.components.get_results.saveAsImage import getResults
@@ -16,7 +23,7 @@ from vectordb_bench.metric import QURIES_PER_DOLLAR_METRIC
 def main():
     # set page config
     initResultsPageConfig(st)
-    
+
     # header
     drawHeaderIcon(st)
 
@@ -57,7 +64,8 @@ def main():
                 dataWithMetric.append(d)
         if len(dataWithMetric) > 0:
             chartContainer = st.expander(caseName, True)
-            drawMetricChart(data, metric, chartContainer)
+            key = f"{caseName}-{metric}"
+            drawMetricChart(data, metric, chartContainer, key=key)
 
     # footer
     footer(st.container())

vectordb_bench/frontend/vdb_benchmark.py

@@ -1,8 +1,13 @@
 import streamlit as st
 from vectordb_bench.frontend.components.check_results.footer import footer
-from vectordb_bench.frontend.components.check_results.stPageConfig import initResultsPageConfig
+from vectordb_bench.frontend.components.check_results.stPageConfig import (
+    initResultsPageConfig,
+)
 from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon
-from vectordb_bench.frontend.components.check_results.nav import NavToQuriesPerDollar, NavToRunTest
+from vectordb_bench.frontend.components.check_results.nav import (
+    NavToQuriesPerDollar,
+    NavToRunTest,
+)
 from vectordb_bench.frontend.components.check_results.charts import drawCharts
 from vectordb_bench.frontend.components.check_results.filters import getshownData
 from vectordb_bench.frontend.components.get_results.saveAsImage import getResults
@@ -20,7 +25,10 @@ def main():
     allResults = benchMarkRunner.get_results()
 
     st.title("Vector Database Benchmark")
-    st.caption("Note that all testing was completed in July 2023, except for the times already noted.")
+    st.caption(
+        "Except for zillizcloud-v2024.1, which was tested in _January 2024_, all other tests were completed before _August 2023_."
+    )
+    st.caption("All tested milvus are in _standalone_ mode.")
 
     # results selector and filter
     resultSelectorContainer = st.sidebar.container()

vectordb_bench/models.py

@@ -1,8 +1,8 @@
 import logging
 import pathlib
-from datetime import date
+from datetime import date, datetime
 from enum import Enum, StrEnum, auto
-from typing import List, Self, Sequence, Set
+from typing import List, Self
 
 import ujson
 
@@ -10,7 +10,6 @@ from .backend.clients import (
     DB,
     DBConfig,
     DBCaseConfig,
-    IndexType,
 )
 from .backend.cases import CaseType
 from .base import BaseModel
@@ -46,8 +45,8 @@ class CaseConfigParamType(Enum):
     numCandidates = "num_candidates"
     lists = "lists"
     probes = "probes"
-    quantizationType = "quantizationType"
-    quantizationRatio = "quantizationRatio"
+    quantizationType = "quantization_type"
+    quantizationRatio = "quantization_ratio"
     m = "m"
     nbits = "nbits"
     intermediate_graph_degree = "intermediate_graph_degree"
@@ -63,6 +62,14 @@ class CaseConfigParamType(Enum):
     level = "level"
     maintenance_work_mem = "maintenance_work_mem"
     max_parallel_workers = "max_parallel_workers"
+    storage_layout = "storage_layout"
+    num_neighbors = "num_neighbors"
+    search_list_size = "search_list_size"
+    max_alpha = "max_alpha"
+    num_dimensions = "num_dimensions"
+    num_bits_per_dimension = "num_bits_per_dimension"
+    query_search_list_size = "query_search_list_size"
+    query_rescore = "query_rescore"
 
 
 class CustomizedCase(BaseModel):
@@ -128,9 +135,14 @@ class TaskConfig(BaseModel):
 
     @property
     def db_name(self):
-        db = self.db.value
+        db_name = f"{self.db.value}"
         db_label = self.db_config.db_label
-        return f"{db}-{db_label}" if db_label else db
+        if db_label:
+            db_name += f"-{db_label}"
+        version = self.db_config.version
+        if version:
+            db_name += f"-{version}"
+        return db_name
 
 
 class ResultLabel(Enum):
@@ -151,16 +163,20 @@ class TestResult(BaseModel):
     results: list[CaseResult]
 
     file_fmt: str = "result_{}_{}_{}.json"  # result_20230718_statndard_milvus.json
+    timestamp: float = 0.0
 
     def flush(self):
         db2case = self.get_db_results()
-
+        timestamp = datetime.combine(date.today(), datetime.min.time()).timestamp()
         result_root = config.RESULTS_LOCAL_DIR
         for db, result in db2case.items():
             self.write_db_file(
                 result_dir=result_root.joinpath(db.value),
                 partial=TestResult(
-                    run_id=self.run_id, task_label=self.task_label, results=result
+                    run_id=self.run_id,
+                    task_label=self.task_label,
+                    results=result,
+                    timestamp=timestamp,
                 ),
                 db=db.value.lower(),
             )