benchbox 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

benchbox/core/results/database.py

@@ -53,7 +53,7 @@ class StoredResult:
     geometric_mean_ms: float | None
     power_at_size: float | None
     throughput_at_size: float | None
-    qphh_at_size: float | None
+    qph_at_size: float | None
     total_cost: float | None
     validation_status: str
     config_hash: str
@@ -77,7 +77,7 @@ class StoredResult:
             geometric_mean_ms=row["geometric_mean_ms"],
             power_at_size=row["power_at_size"],
             throughput_at_size=row["throughput_at_size"],
-            qphh_at_size=row["qphh_at_size"],
+            qph_at_size=row["qph_at_size"],
             total_cost=row["total_cost"],
             validation_status=row["validation_status"],
             config_hash=row["config_hash"],
@@ -238,7 +238,7 @@ class ResultDatabase:
                 geometric_mean_ms REAL,
                 power_at_size REAL,
                 throughput_at_size REAL,
-                qphh_at_size REAL,
+                qph_at_size REAL,
                 total_cost REAL,
                 validation_status TEXT NOT NULL,
                 config_hash TEXT NOT NULL,
@@ -358,7 +358,7 @@ class ResultDatabase:
                     execution_id, platform, platform_version, benchmark, scale_factor,
                     timestamp, duration_seconds, total_queries, successful_queries,
                     failed_queries, geometric_mean_ms, power_at_size, throughput_at_size,
-                    qphh_at_size, total_cost, validation_status, config_hash, metadata_json
+                    qph_at_size, total_cost, validation_status, config_hash, metadata_json
                 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             """,
                 (
@@ -375,7 +375,7 @@ class ResultDatabase:
                     geometric_mean_ms,
                     result.power_at_size,
                     result.throughput_at_size,
-                    result.qphh_at_size,
+                    result.qph_at_size,
                     total_cost,
                     result.validation_status,
                     config_hash,

benchbox/core/results/exporter.py

@@ -17,7 +17,7 @@ import csv
 import io
 import json
 import logging
-from collections.abc import Iterable, Mapping
+from collections.abc import Iterable
 from datetime import datetime
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Union
@@ -36,6 +36,7 @@ from benchbox.core.results.anonymization import (
     AnonymizationManager,
 )
 from benchbox.core.results.models import BenchmarkResults
+from benchbox.core.results.normalizer import get_query_map, normalize_result_dict
 from benchbox.core.results.schema import (
     SchemaV2ValidationError,
     SchemaV2Validator,
@@ -177,21 +178,32 @@ class ResultExporter:
         return exported_files
 
     def _generate_filename_base(self, result: ResultLike, timestamp: str) -> str:
-        """Generate base filename for exports."""
-        short_name = getattr(result, "benchmark_id", None) or getattr(result, "benchmark_name", "unknown")
-        platform = getattr(result, "platform", "unknown")
-
-        try:
-            from benchbox.utils.scale_factor import format_scale_factor
+        """Generate base filename for exports.
 
-            scale_factor = format_scale_factor(getattr(result, "scale_factor", 1.0))
-        except Exception:
-            scale_factor = f"sf{getattr(result, 'scale_factor', 1.0)}"
+        Legacy helper retained for compatibility; delegates to centralized
+        filename builder.
+        """
+        from benchbox.core.results.filenames import build_result_filename_base
 
+        benchmark_id = getattr(result, "benchmark_id", None) or getattr(result, "benchmark_name", "unknown")
+        platform = getattr(result, "platform", "unknown")
+        scale_factor = getattr(result, "scale_factor", 1.0)
         exec_id = getattr(result, "execution_id", None)
-        if exec_id:
-            return f"{str(short_name).lower()}_{scale_factor}_{str(platform).lower()}_{timestamp}_{exec_id}"
-        return f"{str(short_name).lower()}_{scale_factor}_{str(platform).lower()}_{timestamp}"
+
+        # Extract mode from execution_context if available
+        mode = None
+        exec_ctx = getattr(result, "execution_context", None)
+        if isinstance(exec_ctx, dict):
+            mode = exec_ctx.get("mode") or exec_ctx.get("execution_mode")
+
+        return build_result_filename_base(
+            benchmark_id=str(benchmark_id).lower(),
+            scale_factor=scale_factor,
+            platform=str(platform).lower(),
+            timestamp=timestamp,
+            execution_id=exec_id,
+            mode=mode,
+        )
 
     def _export_json_v2(self, result: ResultLike, filename_base: str) -> Path:
         """Export result to JSON using schema v2.0 with companion files."""
@@ -506,12 +518,12 @@ class ResultExporter:
                 # Detect schema version
                 version = data.get("version") or data.get("schema_version", "unknown")
 
-                if version == "2.0":
-                    # Schema v2.0 format
+                if version in ("2.0", "2.1"):
+                    # Schema v2.x format
                     results.append(
                         {
                             "file": json_file,
-                            "version": "2.0",
+                            "version": version,
                             "benchmark": data.get("benchmark", {}).get("name", "Unknown"),
                             "platform": data.get("platform", {}).get("name", "Unknown"),
                             "scale_factor": data.get("benchmark", {}).get("scale_factor", 1.0),
@@ -625,9 +637,9 @@ class ResultExporter:
         baseline_version = baseline_result.get("version", "unknown")
         current_version = current_result.get("version", "unknown")
 
-        # Extract metrics based on schema version
-        perf_baseline = self._extract_performance_metrics(baseline_data, baseline_version)
-        perf_current = self._extract_performance_metrics(current_data, current_version)
+        # Extract metrics using schema-agnostic normalizer
+        perf_baseline = self._extract_performance_metrics(baseline_data)
+        perf_current = self._extract_performance_metrics(current_data)
 
         comparison: dict[str, Any] = {
             "baseline_file": str(baseline_path),
@@ -652,8 +664,8 @@ class ResultExporter:
                 }
 
         # Compare individual queries
-        baseline_queries = self._extract_query_map(baseline_data, baseline_version)
-        current_queries = self._extract_query_map(current_data, current_version)
+        baseline_queries = self._extract_query_map(baseline_data)
+        current_queries = self._extract_query_map(current_data)
 
         for query_id, baseline_query in baseline_queries.items():
             current_query = current_queries.get(query_id)
@@ -690,85 +702,35 @@ class ResultExporter:
 
         return comparison
 
-    def _extract_performance_metrics(self, data: dict[str, Any], version: str) -> dict[str, Any]:
-        """Extract performance metrics from result data."""
-        if version == "2.0":
-            # Schema v2.0 format
-            summary = data.get("summary", {})
-            timing = summary.get("timing", {})
-            queries = summary.get("queries", {})
+    def _extract_performance_metrics(self, data: dict[str, Any]) -> dict[str, Any]:
+        """Extract performance metrics from result data.
 
-            return {
-                "total_queries": queries.get("total", 0),
-                "successful_queries": queries.get("passed", 0),
-                "failed_queries": queries.get("failed", 0),
-                "total_execution_time": timing.get("total_ms", 0) / 1000,
-                "average_query_time": timing.get("avg_ms", 0) / 1000,
-            }
-        else:
-            # Legacy v1.x format
-            results_block = data.get("results", {})
-            if not isinstance(results_block, Mapping):
-                return {
-                    "total_queries": 0,
-                    "successful_queries": 0,
-                    "failed_queries": 0,
-                    "total_execution_time": 0.0,
-                    "average_query_time": 0.0,
-                }
+        Uses the shared normalizer for schema-agnostic extraction.
+        """
+        normalized = normalize_result_dict(data)
+        return {
+            "total_queries": normalized.total_queries,
+            "successful_queries": normalized.passed_queries,
+            "failed_queries": normalized.failed_queries,
+            "total_execution_time": (normalized.total_time_ms or 0) / 1000,
+            "average_query_time": (normalized.avg_time_ms or 0) / 1000,
+        }
 
-            queries_block = results_block.get("queries", {})
-            timing_block = results_block.get("timing", {})
+    def _extract_query_map(self, data: dict[str, Any]) -> dict[str, dict[str, Any]]:
+        """Extract query results as a map from query ID to query data.
 
-            return {
-                "total_queries": queries_block.get("total", 0) if isinstance(queries_block, Mapping) else 0,
-                "successful_queries": queries_block.get("successful", 0) if isinstance(queries_block, Mapping) else 0,
-                "failed_queries": queries_block.get("failed", 0) if isinstance(queries_block, Mapping) else 0,
-                "total_execution_time": (timing_block.get("total_ms", 0) / 1000)
-                if isinstance(timing_block, Mapping)
-                else 0.0,
-                "average_query_time": (timing_block.get("avg_ms", 0) / 1000)
-                if isinstance(timing_block, Mapping)
-                else 0.0,
+        Uses the shared normalizer for schema-agnostic extraction.
+        """
+        normalized = normalize_result_dict(data)
+        query_map = get_query_map(normalized)
+        return {
+            query_id: {
+                "query_id": query_id,
+                "execution_time_ms": q.execution_time_ms or 0,
+                "rows_returned": q.rows_returned,
             }
-
-    def _extract_query_map(self, data: dict[str, Any], version: str) -> dict[str, dict[str, Any]]:
-        """Extract query results as a map from query ID to query data."""
-        if version == "2.0":
-            # Schema v2.0 format - queries is a list
-            queries = data.get("queries", [])
-            result = {}
-            for q in queries:
-                query_id = q.get("id")
-                if query_id:
-                    result[query_id] = {
-                        "query_id": query_id,
-                        "execution_time_ms": q.get("ms", 0),
-                        "rows_returned": q.get("rows"),
-                    }
-            return result
-        else:
-            # Legacy v1.x format
-            results_block = data.get("results", {})
-            if not isinstance(results_block, Mapping):
-                return {}
-
-            queries_block = results_block.get("queries", {})
-            if not isinstance(queries_block, Mapping):
-                return {}
-
-            details = queries_block.get("details", [])
-            result = {}
-            for item in details:
-                if isinstance(item, dict):
-                    query_id = item.get("id") or item.get("query_id")
-                    if query_id:
-                        result[str(query_id)] = {
-                            "query_id": query_id,
-                            "execution_time_ms": item.get("execution_time_ms", 0),
-                            "rows_returned": item.get("rows_returned"),
-                        }
-            return result
+            for query_id, q in query_map.items()
+        }
 
     def _assess_performance_change(self, performance_changes: dict[str, Any]) -> str:
         """Assess overall performance change."""

benchbox/core/results/filenames.py

@@ -0,0 +1,102 @@
+"""Result filename utilities.
+
+Centralizes result filename construction so all exporters and CLI paths
+use the same format.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from benchbox.utils.scale_factor import format_scale_factor
+
+
+def normalize_platform_for_filename(platform: str) -> str:
+    """Normalize platform name for use in filenames.
+
+    - Strips mode suffixes (-df, -sql) to avoid duplication
+    - Replaces hyphens with underscores for consistency
+    - Lowercases the platform name
+    """
+    base = platform.lower()
+    for suffix in ("-df", "-sql"):
+        if base.endswith(suffix):
+            base = base[: -len(suffix)]
+            break
+    return base.replace("-", "_")
+
+
+def abbreviate_mode_for_filename(mode: str) -> str:
+    """Abbreviate execution mode for filenames.
+
+    - 'dataframe' -> 'df'
+    - 'sql' -> 'sql'
+    - 'data_only' -> 'data'
+    """
+    if mode == "dataframe":
+        return "df"
+    if mode == "data_only":
+        return "data"
+    return mode
+
+
+def build_result_filename_base(
+    benchmark_id: str,
+    scale_factor: float,
+    platform: str,
+    timestamp: str | datetime,
+    execution_id: str | None = None,
+    mode: str | None = None,
+) -> str:
+    """Build a consistent base filename for result exports (no extension).
+
+    Args:
+        benchmark_id: Benchmark identifier (e.g., 'tpch', 'tpcds')
+        scale_factor: Scale factor value
+        platform: Platform name (e.g., 'duckdb', 'datafusion-df')
+        timestamp: Timestamp string or datetime object
+        execution_id: Optional execution identifier
+        mode: Optional execution mode ('sql', 'dataframe', 'data_only').
+              When provided, includes normalized platform + abbreviated mode.
+    """
+    if isinstance(timestamp, datetime):
+        timestamp_str = timestamp.strftime("%Y%m%d_%H%M%S")
+    else:
+        timestamp_str = str(timestamp)
+
+    sf_str = format_scale_factor(scale_factor)
+
+    # Normalize platform and include mode if provided
+    if mode:
+        normalized_platform = normalize_platform_for_filename(platform)
+        abbrev_mode = abbreviate_mode_for_filename(mode)
+        platform_part = f"{normalized_platform}_{abbrev_mode}"
+    else:
+        # Legacy behavior: just replace hyphens with underscores
+        platform_part = platform.lower().replace("-", "_")
+
+    base = f"{benchmark_id}_{sf_str}_{platform_part}_{timestamp_str}"
+    if execution_id:
+        base = f"{base}_{execution_id}"
+    return base
+
+
+def build_result_filename(
+    benchmark_id: str,
+    scale_factor: float,
+    platform: str,
+    timestamp: str | datetime,
+    execution_id: str | None = None,
+    extension: str = "json",
+    mode: str | None = None,
+) -> str:
+    """Build a consistent filename for result exports (with extension)."""
+    base = build_result_filename_base(
+        benchmark_id=benchmark_id,
+        scale_factor=scale_factor,
+        platform=platform,
+        timestamp=timestamp,
+        execution_id=execution_id,
+        mode=mode,
+    )
+    return f"{base}.{extension}"

benchbox/core/results/loader.py

@@ -68,9 +68,9 @@ def find_latest_result(
             with open(filepath) as f:
                 data = json.load(f)
 
-            # Only consider v2.0 files
+            # Only consider v2.0/v2.1 files
             version = data.get("version")
-            if version != "2.0":
+            if version not in ("2.0", "2.1"):
                 continue
 
             # Extract metadata for filtering (v2.0 format)
@@ -133,13 +133,13 @@ def load_result_file(filepath: Path | str) -> tuple[BenchmarkResults, dict[str,
     except OSError as e:
         raise ResultLoadError(f"Failed to read result file: {e}") from e
 
-    # Check schema version - only v2.0 is supported
+    # Check schema version - v2.0 and v2.1 are supported
     version = data.get("version")
-    if version != "2.0":
+    if version not in ("2.0", "2.1"):
         legacy_version = data.get("schema_version", version)
         raise UnsupportedSchemaError(
             f"Unsupported schema version: {legacy_version}. "
-            f"Only schema v2.0 is supported. "
+            f"Only schema v2.0 and v2.1 are supported. "
             f"Please re-export the result using the current version of BenchBox."
         )
 
@@ -232,10 +232,11 @@ def reconstruct_benchmark_results(
     tpc_section = summary_section.get("tpc_metrics", {})
     power_at_size = tpc_section.get("power_at_size")
     throughput_at_size = tpc_section.get("throughput_at_size")
-    qphh_at_size = tpc_section.get("qphh_at_size")
+    # Load composite metric (either qphh_at_size or qphds_at_size depending on benchmark)
+    qph_at_size = tpc_section.get("qphh_at_size") or tpc_section.get("qphds_at_size")
 
-    # Geometric mean can be in either timing section (computed) or tpc_metrics (explicit)
-    geometric_mean_ms = timing_section.get("geometric_mean_ms") or tpc_section.get("geometric_mean_ms")
+    # Geometric mean is in timing section (computed from query times)
+    geometric_mean_ms = timing_section.get("geometric_mean_ms")
     geometric_mean_execution_time = geometric_mean_ms / 1000.0 if geometric_mean_ms else None
 
     # Extract table statistics
@@ -329,7 +330,7 @@ def reconstruct_benchmark_results(
         # TPC metrics
         power_at_size=power_at_size,
         throughput_at_size=throughput_at_size,
-        qphh_at_size=qphh_at_size,
+        qph_at_size=qph_at_size,
         geometric_mean_execution_time=geometric_mean_execution_time,
         # Metadata
         test_execution_type=benchmark_section.get("mode", "standard"),

benchbox/core/results/metrics.py

@@ -0,0 +1,211 @@
+"""TPC benchmark metrics calculators.
+
+This module provides standardized calculation of TPC-compliant benchmark metrics
+including Power@Size, Throughput@Size, and QphH/QphDS composite metrics.
+
+Copyright 2026 Joe Harris / BenchBox Project
+
+Licensed under the MIT License. See LICENSE file in the project root for details.
+"""
+
+from __future__ import annotations
+
+import math
+import statistics
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+
+class TPCMetricsCalculator:
+    """Calculate TPC-compliant benchmark metrics.
+
+    Implements metrics as defined in TPC-H and TPC-DS specifications:
+    - Power@Size: Single-stream query throughput metric
+    - Throughput@Size: Multi-stream concurrent throughput metric
+    - QphH/QphDS: Composite metric combining power and throughput
+    """
+
+    @staticmethod
+    def calculate_power_at_size(
+        query_times_seconds: Sequence[float],
+        scale_factor: float,
+    ) -> float:
+        """Calculate Power@Size metric (TPC-H/TPC-DS specification).
+
+        Power@Size measures single-stream query performance:
+        Power@Size = (SF * 3600) / geometric_mean(query_times)
+
+        This represents queries per hour at the given scale factor.
+
+        Args:
+            query_times_seconds: List of query execution times in seconds
+            scale_factor: Benchmark scale factor (e.g., 0.01, 1, 10, 100)
+
+        Returns:
+            Power@Size metric value, or 0.0 if calculation not possible
+        """
+        if not query_times_seconds:
+            return 0.0
+
+        # Filter out zero/negative times which would break geometric mean
+        valid_times = [t for t in query_times_seconds if t > 0]
+        if not valid_times:
+            return 0.0
+
+        geometric_mean = statistics.geometric_mean(valid_times)
+        if geometric_mean <= 0:
+            return 0.0
+
+        return (scale_factor * 3600) / geometric_mean
+
+    @staticmethod
+    def calculate_throughput_at_size(
+        total_queries: int,
+        total_time_seconds: float,
+        scale_factor: float,
+        num_streams: int = 1,
+    ) -> float:
+        """Calculate Throughput@Size metric (TPC-H/TPC-DS specification).
+
+        Throughput@Size measures concurrent multi-stream performance:
+        Throughput@Size = (S * Q * SF * 3600) / T_s
+
+        Where:
+        - S = number of concurrent streams
+        - Q = number of queries per stream
+        - SF = scale factor
+        - T_s = total elapsed time in seconds
+
+        Args:
+            total_queries: Total queries executed across all streams
+            total_time_seconds: Total elapsed wall-clock time
+            scale_factor: Benchmark scale factor
+            num_streams: Number of concurrent streams
+
+        Returns:
+            Throughput@Size metric value, or 0.0 if calculation not possible
+        """
+        if total_time_seconds <= 0 or num_streams <= 0:
+            return 0.0
+
+        # Per TPC spec: (Streams * QueriesPerStream * SF * 3600) / TotalTime
+        # But we have total_queries already = Streams * QueriesPerStream
+        return (total_queries * scale_factor * 3600) / total_time_seconds
+
+    @staticmethod
+    def calculate_qph(power_at_size: float, throughput_at_size: float) -> float:
+        """Calculate composite QphH/QphDS metric.
+
+        The composite metric is the geometric mean of Power and Throughput:
+        QphH = sqrt(Power@Size * Throughput@Size)
+
+        Args:
+            power_at_size: Power@Size metric value
+            throughput_at_size: Throughput@Size metric value
+
+        Returns:
+            Composite QphH/QphDS metric, or 0.0 if either input is invalid
+        """
+        if power_at_size <= 0 or throughput_at_size <= 0:
+            return 0.0
+
+        return statistics.geometric_mean([power_at_size, throughput_at_size])
+
+    @staticmethod
+    def calculate_geometric_mean(times: Sequence[float]) -> float:
+        """Calculate geometric mean of execution times.
+
+        Args:
+            times: Sequence of execution times (in any unit)
+
+        Returns:
+            Geometric mean of times, or 0.0 if not calculable
+        """
+        if not times:
+            return 0.0
+
+        valid_times = [t for t in times if t > 0]
+        if not valid_times:
+            return 0.0
+
+        return statistics.geometric_mean(valid_times)
+
+
+class TimingStatsCalculator:
+    """Calculate timing statistics for query results."""
+
+    @staticmethod
+    def calculate(times_ms: Sequence[float]) -> dict[str, float]:
+        """Calculate comprehensive timing statistics.
+
+        Args:
+            times_ms: List of execution times in milliseconds
+
+        Returns:
+            Dictionary containing:
+            - total_ms: Sum of all times
+            - avg_ms: Arithmetic mean
+            - min_ms: Minimum time
+            - max_ms: Maximum time
+            - geometric_mean_ms: Geometric mean
+            - stdev_ms: Standard deviation (0 if single value)
+            - p50_ms: 50th percentile (median)
+            - p90_ms: 90th percentile
+            - p95_ms: 95th percentile
+            - p99_ms: 99th percentile
+        """
+        if not times_ms:
+            return {}
+
+        times_list = list(times_ms)
+        sorted_times = sorted(times_list)
+        n = len(sorted_times)
+
+        def percentile(p: float) -> float:
+            """Calculate percentile using nearest-rank method."""
+            if n == 0:
+                return 0.0
+            k = max(1, math.ceil(n * p))
+            k = min(k, n)  # Clamp to valid rank
+            k -= 1  # Convert rank to 0-based index
+            return sorted_times[k]
+
+        # Calculate geometric mean (only for positive values)
+        positive_times = [t for t in times_list if t > 0]
+        geom_mean = statistics.geometric_mean(positive_times) if positive_times else 0.0
+
+        return {
+            "total_ms": sum(times_list),
+            "avg_ms": statistics.mean(times_list),
+            "min_ms": min(times_list),
+            "max_ms": max(times_list),
+            "geometric_mean_ms": geom_mean,
+            "stdev_ms": statistics.stdev(times_list) if n > 1 else 0.0,
+            "p50_ms": percentile(0.50),
+            "p90_ms": percentile(0.90),
+            "p95_ms": percentile(0.95),
+            "p99_ms": percentile(0.99),
+        }
+
+    @staticmethod
+    def calculate_seconds(times_seconds: Sequence[float]) -> dict[str, float]:
+        """Calculate timing statistics with times in seconds.
+
+        Convenience method that returns results in seconds instead of milliseconds.
+
+        Args:
+            times_seconds: List of execution times in seconds
+
+        Returns:
+            Same structure as calculate() but with _s suffix instead of _ms
+        """
+        if not times_seconds:
+            return {}
+
+        times_ms = [t * 1000 for t in times_seconds]
+        stats_ms = TimingStatsCalculator.calculate(times_ms)
+
+        # Convert back to seconds
+        return {key.replace("_ms", "_s"): value / 1000 for key, value in stats_ms.items()}

benchbox/core/results/models.py

@@ -215,7 +215,7 @@ class BenchmarkResults:
     test_execution_type: str = "standard"
     power_at_size: float | None = None
     throughput_at_size: float | None = None
-    qphh_at_size: float | None = None
+    qph_at_size: float | None = None  # TPC composite metric (QphH for TPC-H, QphDS for TPC-DS)
     geometric_mean_execution_time: float | None = None
     # Validation and metadata
     validation_status: str = "PASSED"
@@ -256,6 +256,8 @@ class BenchmarkResults:
     total_plan_capture_time_ms: float = 0.0  # Total time spent on plan capture
     avg_plan_capture_overhead_pct: float = 0.0  # Average overhead as % of query time
     max_plan_capture_time_ms: float = 0.0  # Maximum single capture time
+    # Execution context for reproducibility (captures CLI/MCP/API params)
+    execution_context: dict[str, Any] | None = None
 
     @property
     def benchmark_id(self) -> str: