vectordb-bench 0.0.29__py3-none-any.whl → 1.0.0__py3-none-any.whl

vectordb_bench/frontend/components/streaming/data.py

@@ -0,0 +1,62 @@
+from enum import StrEnum
+from dataclasses import dataclass
+
+
+class DisplayedMetric(StrEnum):
+    db = "db"
+    db_name = "db_name"
+    search_stage = "search_stage"
+    search_time = "search_time"
+    qps = "qps"
+    recall = "recall"
+    ndcg = "ndcg"
+    adjusted_recall = "adjusted_recall"
+    adjusted_ndcg = "adjusted_ndcg"
+    latency_p99 = "latency_p99"
+    # st_ideal_insert_duration = "st_ideal_insert_duration"
+    # st_search_time_list = "st_search_time_list"
+    insert_duration = "insert_duration"
+    optimize_duration = "optimize_duration"
+
+
+@dataclass
+class StreamingData:
+    db: str
+    db_name: str
+    search_stage: int
+    search_time: float
+    qps: float
+    recall: float
+    ndcg: float
+    adjusted_recall: float
+    adjusted_ndcg: float
+    latency_p99: float
+    ideal_insert_duration: int
+    insert_duration: float
+    optimize_duration: float
+
+    @property
+    def optimized(self) -> bool:
+        return self.search_stage > 100
+
+
+def get_streaming_data(data) -> list[StreamingData]:
+    return [
+        StreamingData(
+            db=d["db"],
+            db_name=d["db_name"],
+            search_stage=search_stage,
+            search_time=d["st_search_time_list"][i],
+            qps=d["st_max_qps_list_list"][i],
+            recall=d["st_recall_list"][i],
+            ndcg=d["st_ndcg_list"][i],
+            adjusted_recall=round(d["st_recall_list"][i] / min(search_stage, 100) * 100, 4),
+            adjusted_ndcg=round(d["st_ndcg_list"][i] / min(search_stage, 100) * 100, 4),
+            latency_p99=round(d["st_serial_latency_p99_list"][i] * 1000, 2),
+            ideal_insert_duration=d["st_ideal_insert_duration"],
+            insert_duration=d["insert_duration"],
+            optimize_duration=d["optimize_duration"],
+        )
+        for d in data
+        for i, search_stage in enumerate(d["st_search_stage_list"])
+    ]

vectordb_bench/frontend/components/tables/data.py

@@ -24,7 +24,7 @@ def formatData(caseResults: list[CaseResult]):
         db = caseResult.task_config.db.value
         db_label = caseResult.task_config.db_config.db_label
         case_config = caseResult.task_config.case_config
-        case = case_config.case_id.case_cls()
+        case = case_config.case
         filter_rate = case.filter_rate
         dataset = case.dataset.data.name
         metrics = asdict(caseResult.metrics)

vectordb_bench/frontend/components/welcome/explainPrams.py

@@ -0,0 +1,66 @@
+def explainPrams(st):
+    st.markdown("## descriptions")
+    st.markdown("### 1.Overview")
+    st.markdown(
+        """
+- **VectorDBBench** is an open-source benchmarking tool designed specifically for vector databases. Its main features include:
+    - (1) An easy-to-use **web UI** for configuration of tests and visual analysis of results.
+    - (2) A comprehensive set of **standards for testing and metric collection**.
+    - (3) Support for **various scenarios**, including additional support for **Filter** and **Streaming** based on standard tests.
+- VectorDBBench embraces open-source and welcome contributions of code and test result submissions. The testing process and extended scenarios of VectorDBBench, as well as the intention behind our design will be introduced as follows.
+"""
+    )
+    st.markdown("### 2.Dataset")
+    st.markdown(
+        """
+- We provide two embedding datasets:
+    - (1)*[Cohere 768dim](https://huggingface.co/datasets/Cohere/wikipedia-22-12)*, generated using the **Cohere** model based on the Wikipedia corpus. 
+    - (2)*[Cohere 1024dim](https://huggingface.co/datasets/Cohere/beir-embed-english-v3)*, generated using the **Cohere** embed-english-v3.0 model based on the bioasq corpus.
+    - (3)*OpenAI 1536dim*, generated using the **OpenAI** model based on the [C4 corpus](https://huggingface.co/datasets/legacy-datasets/c4).
+"""
+    )
+    st.markdown("### 3.Standard Test")
+    st.markdown(
+        """
+The test is actually divided into 3 sub-processes
+- **3.1 Test Part 1 - Load (Insert + Optimize)**
+    - (1) Use a single process to perform serial inserts until all data is inserted, and record the time taken as **insert_duration**.
+    - (2) For most vector databases, index construction requires additional time to optimize to achieve an optimal state, and record the time taken as **optimize_duration**.
+    - (3) **Load_duration (insert_duration + optimize_duration)** can be understood as the time from the start of insertion until the database is ready to query.
+        - load_duration can serve as a reference for the insert capability of a vector database to some extent. However, it should be noted that some vector databases may perform better under **concurrent insert operations**.
+- **3.2 Test Part 2 - Serial Search Test**
+    - (1) Use a single process to perform serial searches, record the results and time taken for each search, and calculate **recall** and **latency**.
+    - (2) **Recall**: For vector databases, most searches are approximately nearest neighbor(ANN) searches rather than perfectly accurate results. In production environments, commonly targeted recall rates are 0.9 or 0.95.
+        - Note that there is a **trade-off** between **accuracy** and **search performance**. By adjusting parameters, it is possible to sacrifice some accuracy in exchange for better performance. We recommend comparing performance while ensuring that the recall rates remain reasonably close.
+    - (3) **Latency**:**p99** rather than average. **latency_p99** focuses on **the slowest 1% of requests**. In many high-demand applications, ensuring that most user requests stay within acceptable latency limits is critical, whereas **latency_avg** can be skewed by faster requests.
+        - **serial_latency** can serve as a reference for a database's search capability to some extent. However, serial_latency is significantly affected by network conditions. We recommend running the test client and database server within the same local network.
+- **3.3 Test Part 3 - Concurrent Search Test**
+    - (1) Create multiple processes, each perform serial searches independently to test the database's **maximum throughput(max-qps)**.
+    - (2) Since different databases may reach peak throughput under different conditions, we conduct multiple test rounds. The number of processes **starts at 1 by default and gradually increases up to 80**, with each test group running for **30 seconds**.
+        - Detailed latency and QPS metrics at different concurrency levels can be viewed on the <a href="concurrent" target="_self" style="text-decoration: none;">*concurrent*</a> page.
+        - The highest recorded QPS value from these tests will be selected as the final max-qps.
+""",
+        unsafe_allow_html=True,
+    )
+    st.markdown("### 4.Filter Search Test")
+    st.markdown(
+        """
+- Compared to the Standard Test, the **Filter Search** introduces additional scalar constraints (e.g. **color == red**) during the Search Test. Different **filter_ratios** present varying levels of challenge to the VectorDB's search performance.
+- We provide an additional **string column** containing 10 labels with different distribution ratios (50%,20%,10%,5%,2%,1%,0.5%,0.2%,0.1%). For each label, we conduct both a **Serial Test** and a **Concurrency Test** to observe the VectorDB's performance in terms of **QPS, latency, and recall** under different filtering conditions.
+"""
+    )
+    st.markdown("### 5.Streaming Search Test")
+    st.markdown(
+        """
+Different from Standard's load and search separation, Streaming Search Test primarily focuses on **search performance during the insertion process**. 
+Different **base dataset sizes** and varying **insertion rates** set distinct challenges to the VectorDB's search capabilities.
+VectorDBBench will send insert requests at a **fixed rate**, maintaining consistent insertion pressure. The search test consists of three steps as follows:
+- 1.**Streaming Search** 
+    - Users can configure **multiple search stages**. When the inserted data volume reaches a specified stage, a **Serial Test** and a **Concurrent Test** will be conducted, recording qps, latency, and recall performance.
+- 2.**Streaming Final Search**
+    - After all of the data is inserted, a Serial Test and a Concurrent Test are immediately performed, recording qps, latency, and recall performance. 
+        - Note: at this time, the insertion pressure drops to zero since data insertion is complete.
+- 3.**Optimized Search (Optional)**
+    - Users can optionally perform an additional optimization step followed by a Serial Test and a Concurrent Test, recording qps, latency, and recall performance. This step **compares performance in Streaming section with the theoretically optimal performance**.
+"""
+    )

vectordb_bench/frontend/components/welcome/pagestyle.py

@@ -0,0 +1,106 @@
+def pagestyle():
+    html_content = """
+    <style>
+    .grid-container {
+        display: grid;
+        grid-template-columns: repeat(3, 1fr);  
+        grid-template-rows: repeat(3, auto);   
+        gap: 20px;
+        padding: 20px 0;
+    }
+
+    .title-row {
+    grid-column: 1 / 4;  
+    text-align: left;  
+    margin: 20px 0;    
+    }
+
+    .title-row h2 {
+        font-size: 35px;     
+        color: #333;       
+        font-weight: 600;   
+    }
+
+    .last-row {
+    grid-column: 1 / 7;  
+    display: grid;
+    grid-template-columns: repeat(6, 1fr);  
+    gap: 40px;
+    }
+
+    .last-row > :nth-child(1) {
+        grid-column: 2 / 4;
+    }
+
+    .last-row > :nth-child(2) {
+        grid-column: 4 / 6;
+            gap: 40px;
+        }
+    .section-card {
+        width: 100%;
+        height: 350px;
+        padding: 15px;
+        border-radius: 10px;
+        background-color: #f0f2f6;
+        box-shadow: 0 4px 6px rgba(0,0,0,0.1);
+        transition: transform 0.3s ease, box-shadow 0.3s ease;
+        text-align: center;
+        overflow: hidden;
+        cursor: pointer;
+        display: flex;
+        flex-direction: column;
+    }
+    .section-card:hover {
+        transform: scale(1.05);
+        box-shadow: 0 10px 20px rgba(0,0,0,0.2);
+        z-index: 100;
+    }
+    .section-image {
+        width: 100%;
+        height: 185px;
+        object-fit: cover;
+        object-position: 0 0;
+        border-radius: 5px;
+        margin-bottom: 10px;
+    }
+    .section-title {
+        font-size: 18px;
+        font-weight: bold;
+        margin-bottom: 10px;
+        color: #262730;
+    }
+    .section-description {
+        font-size: 14px;
+        color: #555;
+        height: 80px;
+        overflow-y: auto;
+        margin-bottom: 10px;
+    }
+    .scroll-container {
+        width: 100%;
+        overflow-x: auto;
+        white-space: nowrap;
+        margin-top: auto;
+        padding: 10px 0;
+        border-top: 1px solid #eee;
+    }
+    .scroll-content {
+        display: inline-block;
+        white-space: nowrap;
+        padding: 0 10px;
+    }
+    .scroll-item {
+        display: inline-block;
+        width: 50px;
+        height: 30px;
+        margin-right: 10px;
+        background-color: #ddd;
+        border-radius: 5px;
+        text-align: center;
+        line-height: 30px;
+    }
+    </style>
+
+    <div class="grid-container">
+    """
+    return html_content

vectordb_bench/frontend/components/welcome/welcomePrams.py

@@ -0,0 +1,147 @@
+import base64
+from PIL import Image
+from io import BytesIO
+import os
+
+from vectordb_bench.frontend.components.welcome.pagestyle import pagestyle
+
+
+def get_image_as_base64(image_path):
+    try:
+        if image_path.startswith("http"):
+            return image_path
+
+        path = os.path.expanduser(image_path)
+        img = Image.open(path)
+        buffered = BytesIO()
+        img.save(buffered, format="PNG")
+        return f"data:image/png;base64,{base64.b64encode(buffered.getvalue()).decode()}"
+    except Exception as e:
+        return None
+
+
+def welcomePrams(st):
+    st.title("Welcome to VectorDB Benchmark!")
+    options = [
+        {
+            "title": "Standard Test Results",
+            "description": (
+                "<span style='font-size: 17px;'>"
+                "Select a specific run or compare all results side by side to view the results of previous tests."
+                "</span>"
+            ),
+            "image": "fig/homepage/bar-chart.png",
+            "link": "results",
+        },
+        {
+            "title": "Quries Per Dollar",
+            "description": (
+                "<span style='font-size: 17px;'>"
+                "To view the results of quries per dollar.<br> "
+                "(similar to qps in Results) "
+                "</span>"
+            ),
+            "image": "fig/homepage/qp$.png",
+            "link": "quries_per_dollar",
+        },
+        {
+            "title": "Tables",
+            "description": (
+                "<span style='font-size: 17px;'>" "To view the results of differnt datasets in tables." "</span>"
+            ),
+            "image": "fig/homepage/table.png",
+            "link": "tables",
+        },
+        {
+            "title": "Concurrent Performance",
+            "description": (
+                "<span style='font-size: 17px;'>"
+                "To view the variation of qps with latency under different concurrent."
+                "</span>"
+            ),
+            "image": "fig/homepage/concurrent.png",
+            "link": "concurrent",
+        },
+        {
+            "title": "Label Filter Performance",
+            "description": (
+                "<span style='font-size: 17px;'>"
+                "To view the perfomance of datasets under different filter ratios "
+                "</span>"
+            ),
+            "image": "fig/homepage/label_filter.png",
+            "link": "label_filter",
+        },
+        {
+            "title": "Streaming Performance",
+            "description": (
+                "<span style='font-size: 17px;'>"
+                "To view the perfomance of datasets under different search stages and insertion rates. "
+                "</span>"
+            ),
+            "image": "fig/homepage/streaming.png",
+            "link": "streaming",
+        },
+        {
+            "title": "Run Test",
+            "description": (
+                "<span style='font-size: 17px;'>"
+                "Select the databases and cases to test.<br>"
+                "The test results will be displayed in Results."
+                "</span>"
+            ),
+            "image": "fig/homepage/run_test.png",
+            "link": "run_test",
+        },
+        {
+            "title": "Custom Dataset",
+            "description": (
+                "<span style='font-size: 17px;'>"
+                "Define users' own datasets with detailed descriptions of setting each parameter."
+                "</span>"
+            ),
+            "image": "fig/homepage/custom.png",
+            "link": "custom",
+        },
+    ]
+
+    html_content = pagestyle()
+
+    for option in options:
+        option["image"] = get_image_as_base64(option["image"])
+
+    for i, option in enumerate(options[:6]):
+        html_content += f"""
+        <a href="/{option['link']}" target="_self" style="text-decoration: none;">
+            <div class="section-card">
+                <img src="{option['image']}" class="section-image" alt="{option['title']}">
+                <div class="section-title">{option['title']}</div>
+                <div class="section-description">{option['description']}</div>
+            </div>
+        </a>
+        """
+
+    html_content += """
+    </div>
+    <div class="title-row">
+        <h2>Run Your Own Test</h2>
+    </div>
+    <div class="last-row">
+    """
+
+    for option in options[6:8]:
+        html_content += f"""
+        <a href="/{option['link']}" target="_self" style="text-decoration: none;">
+            <div class="section-card">
+                <img src="{option['image']}" class="section-image" alt="{option['title']}">
+                <div class="section-title">{option['title']}</div>
+                <div class="section-description">{option['description']}</div>
+            </div>
+        </a>
+        """
+
+    html_content += """
+    </div>
+    """
+
+    st.html(html_content)