benchbox 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

benchbox/core/results/schema.py

@@ -22,13 +22,70 @@ from dataclasses import asdict, is_dataclass
 from datetime import datetime
 from typing import TYPE_CHECKING, Any
 
+from benchbox.core.results.builder import normalize_benchmark_id
+from benchbox.core.results.query_normalizer import normalize_query_id
+
 if TYPE_CHECKING:
     from benchbox.core.results.models import BenchmarkResults
 
-SCHEMA_VERSION = "2.0"
+SCHEMA_VERSION = "2.1"
 
 logger = logging.getLogger(__name__)
 
+CANONICAL_KEY_ORDER = [
+    "version",
+    "run",
+    "benchmark",
+    "platform",
+    "config",
+    "summary",
+    "phases",
+    "queries",
+    "tables",
+    "validation",
+    "cost",
+    "execution",
+    "environment",
+    "export",
+    "errors",
+]
+
+QUERY_KEY_ORDER = ["id", "ms", "rows", "iter", "stream", "run_type", "status"]
+CONFIG_KEY_ORDER = [
+    "compression",
+    "seed",
+    "phases",
+    "query_subset",
+    "parallelism",
+    "tuning_mode",
+    "tuning_config",
+    "platform_options",
+    "mode",
+    "test_type",
+]
+PHASE_KEY_ORDER = [
+    "data_generation",
+    "schema_creation",
+    "data_loading",
+    "validation",
+    "power_test",
+    "throughput_test",
+]
+
+
+def order_dict(d: dict[str, Any], key_order: list[str]) -> dict[str, Any]:
+    """Return dict with keys ordered for stable JSON diffs."""
+    from collections import OrderedDict
+
+    ordered: OrderedDict[str, Any] = OrderedDict()
+    for key in key_order:
+        if key in d:
+            ordered[key] = d[key]
+    for key in sorted(d.keys()):
+        if key not in ordered:
+            ordered[key] = d[key]
+    return ordered
+
 
 def _normalize_query_result(qr: Any) -> dict[str, Any]:
     """Normalize a query result entry to a dictionary.
@@ -57,6 +114,7 @@ def _normalize_query_result(qr: Any) -> dict[str, Any]:
         "iteration",
         "stream_id",
         "error_message",
+        "run_type",
         "error",
         "error_type",
         "query_plan",
@@ -81,7 +139,7 @@ class SchemaV2Validator:
     """
 
     REQUIRED_KEYS = ("version", "run", "benchmark", "platform", "summary", "queries")
-    OPTIONAL_KEYS = ("environment", "tables", "errors", "cost", "export", "tuning")
+    OPTIONAL_KEYS = ("environment", "tables", "errors", "cost", "export", "tuning", "execution", "config", "phases")
 
     RUN_REQUIRED = ("id", "timestamp", "total_duration_ms", "query_time_ms")
     BENCHMARK_REQUIRED = ("id", "name", "scale_factor")
@@ -95,10 +153,11 @@ class SchemaV2Validator:
         if missing_top:
             raise SchemaV2ValidationError(f"schema v2.0 payload missing keys: {missing_top}")
 
-        # Validate version
-        if payload.get("version") != SCHEMA_VERSION:
+        # Validate version (accept 2.0 and 2.1 as valid)
+        valid_versions = ("2.0", "2.1")
+        if payload.get("version") not in valid_versions:
             raise SchemaV2ValidationError(
-                f"invalid schema version {payload.get('version')} (expected {SCHEMA_VERSION})"
+                f"invalid schema version {payload.get('version')} (expected one of {valid_versions})"
             )
 
         # Validate run block
@@ -165,8 +224,6 @@ def build_result_payload(result: BenchmarkResults) -> dict[str, Any]:
     errors_list: list[dict[str, Any]] = []
 
     # Determine if we have multi-iteration or multi-stream runs
-    has_iterations = False
-    has_streams = False
     iterations_set: set[int] = set()
     streams_set: set[int] = set()
 
@@ -174,21 +231,16 @@ def build_result_payload(result: BenchmarkResults) -> dict[str, Any]:
     normalized_results = [_normalize_query_result(qr) for qr in (result.query_results or [])]
 
     for qr in normalized_results:
-        iteration = qr.get("iteration")
-        stream_id = qr.get("stream_id")
-        if iteration is not None and iteration != 1:
-            has_iterations = True
-            iterations_set.add(iteration)
-        if iteration == 1:
-            iterations_set.add(1)
-        if stream_id is not None and stream_id != 1:
-            has_streams = True
-            streams_set.add(stream_id)
-        if stream_id == 1:
-            streams_set.add(1)
+        iteration = qr.get("iteration", 1)
+        stream_id = qr.get("stream_id", 0)
+        if iteration is not None:
+            iterations_set.add(int(iteration))
+        if stream_id is not None:
+            streams_set.add(int(stream_id))
 
     for qr in normalized_results:
-        query_id = qr.get("query_id", qr.get("id", ""))
+        raw_id = qr.get("query_id") or qr.get("id") or qr.get("query") or ""
+        query_id = normalize_query_id(raw_id)
         status = qr.get("status", "UNKNOWN")
 
         # Get execution time in ms
@@ -197,30 +249,18 @@ def build_result_payload(result: BenchmarkResults) -> dict[str, Any]:
         if exec_time_ms is None and exec_time is not None:
             exec_time_ms = exec_time * 1000
 
-        rows = qr.get("rows_returned")
-
-        if status == "SUCCESS" and exec_time_ms is not None:
-            query_times_ms.append(exec_time_ms)
-
-            # Build compact query entry
-            entry: dict[str, Any] = {"id": str(query_id)}
-            entry["ms"] = round(exec_time_ms, 1)
-            if rows is not None:
-                entry["rows"] = rows
-
-            # Add optional fields only when non-default
-            iteration = qr.get("iteration")
-            if has_iterations and iteration is not None:
-                entry["iter"] = iteration
+        rows = qr.get("rows_returned") or qr.get("rows") or qr.get("result_count")
+        iteration = int(qr.get("iteration", 1))
+        stream_id = int(qr.get("stream_id", 0))
+        run_type = qr.get("run_type")
+        if not run_type:
+            run_type = "warmup" if iteration == 0 else "measurement"
 
-            stream_id = qr.get("stream_id")
-            if has_streams and stream_id is not None:
-                entry["stream"] = stream_id
-
-            queries_list.append(entry)
+        if status == "SUCCESS":
+            if exec_time_ms is not None and run_type == "measurement":
+                query_times_ms.append(exec_time_ms)
         else:
             failed_count += 1
-            # Add to errors array
             error_entry = {
                 "phase": "query",
                 "query_id": str(query_id),
@@ -234,9 +274,23 @@ def build_result_payload(result: BenchmarkResults) -> dict[str, Any]:
             error_entry["message"] = qr.get("error_message") or qr.get("error") or "Query failed"
             errors_list.append(error_entry)
 
+        entry: dict[str, Any] = {"id": str(query_id)}
+        if exec_time_ms is not None:
+            entry["ms"] = round(exec_time_ms, 1)
+        if rows is not None:
+            entry["rows"] = rows
+        entry["iter"] = iteration
+        entry["stream"] = stream_id
+        entry["run_type"] = run_type
+        entry["status"] = status
+
+        queries_list.append(order_dict(entry, QUERY_KEY_ORDER))
+
     # Compute timing statistics
-    total_queries = len(queries_list) + failed_count
-    successful_queries = len(queries_list)
+    measurement_queries = [q for q in queries_list if q.get("run_type") == "measurement"]
+    total_queries = len(measurement_queries)
+    successful_queries = len([q for q in measurement_queries if q.get("status") == "SUCCESS"])
+    failed_count = total_queries - successful_queries
 
     timing_stats = _compute_timing_stats(query_times_ms)
 
@@ -277,29 +331,29 @@ def build_result_payload(result: BenchmarkResults) -> dict[str, Any]:
         "query_time_ms": round(sum(query_times_ms)),
     }
 
-    # Add optional run fields
-    if len(iterations_set) > 1:
-        run["iterations"] = max(iterations_set)
-    if len(streams_set) > 1:
-        run["streams"] = max(streams_set)
+    # Always include iterations and streams
+    run["iterations"] = max(iterations_set) if iterations_set else 1
+    run["streams"] = max(streams_set) if streams_set else 1
     if result.query_subset:
         run["query_subset"] = result.query_subset
 
     # Build benchmark block
+    benchmark_name = _shorten_benchmark_name(result.benchmark_name)
     benchmark: dict[str, Any] = {
         "id": result.benchmark_id,
-        "name": result.benchmark_name,
+        "name": benchmark_name,
         "scale_factor": result.scale_factor,
+        "test_type": result.test_execution_type,
     }
-    if result.test_execution_type and result.test_execution_type != "standard":
-        benchmark["mode"] = result.test_execution_type
 
     # Build platform block
-    platform: dict[str, Any] = {"name": result.platform}
+    platform_name = str(result.platform).replace(" (DataFrame)", "")
+    platform: dict[str, Any] = {"name": platform_name}
     if result.platform_info:
-        version = result.platform_info.get("version")
-        if version:
-            platform["version"] = version
+        version = result.platform_info.get("platform_version") or result.platform_info.get("version")
+        client_version = result.platform_info.get("client_library_version")
+        platform["version"] = version or "unknown"
+        platform["client_version"] = client_version or "unknown"
         variant = result.platform_info.get("variant")
         if variant:
             platform["variant"] = variant
@@ -308,6 +362,10 @@ def build_result_payload(result: BenchmarkResults) -> dict[str, Any]:
         config = _extract_platform_config(result.platform_info)
         if config:
             platform["config"] = config
+    if "version" not in platform:
+        platform["version"] = "unknown"
+    if "client_version" not in platform:
+        platform["client_version"] = "unknown"
 
     # Add tuning summary if available
     tuning = _build_tuning_summary(result)
@@ -320,6 +378,10 @@ def build_result_payload(result: BenchmarkResults) -> dict[str, Any]:
     # Build tables block (compact)
     tables = _build_tables_block(result.table_statistics)
 
+    # Build config and phases blocks
+    config_block = _build_config_block(result)
+    phases_block = _build_phases_block(result)
+
     # Add table loading errors if present
     if result.execution_phases:
         table_errors = _extract_table_errors(result.execution_phases)
@@ -328,10 +390,14 @@ def build_result_payload(result: BenchmarkResults) -> dict[str, Any]:
     # Build the payload
     payload: dict[str, Any] = {
         "version": SCHEMA_VERSION,
-        "run": run,
-        "benchmark": benchmark,
-        "platform": platform,
-        "summary": summary,
+        "run": order_dict(
+            run, ["id", "timestamp", "total_duration_ms", "query_time_ms", "iterations", "streams", "query_subset"]
+        ),
+        "benchmark": order_dict(benchmark, ["id", "name", "scale_factor", "test_type"]),
+        "platform": order_dict(platform, ["name", "version", "client_version", "variant", "config", "tuning"]),
+        "config": order_dict(config_block, CONFIG_KEY_ORDER),
+        "summary": order_dict(summary, ["queries", "timing", "data", "validation", "tpc_metrics"]),
+        "phases": order_dict(phases_block, PHASE_KEY_ORDER),
         "queries": queries_list,
     }
 
@@ -341,6 +407,7 @@ def build_result_payload(result: BenchmarkResults) -> dict[str, Any]:
     if tables:
         payload["tables"] = tables
     if errors_list:
+        errors_list.sort(key=lambda e: (e.get("phase", ""), e.get("query_id", ""), e.get("type", "")))
         payload["errors"] = errors_list
 
     # Add cost summary if available
@@ -352,7 +419,213 @@ def build_result_payload(result: BenchmarkResults) -> dict[str, Any]:
         if cost_block:
             payload["cost"] = cost_block
 
-    return payload
+    # Add execution context for reproducibility (v2.1+)
+    if result.execution_context:
+        exec_block: dict[str, Any] = {}
+        ctx = result.execution_context
+
+        # Always include entry point
+        if ctx.get("entry_point"):
+            exec_block["entry_point"] = ctx["entry_point"]
+
+        # Timestamp
+        if ctx.get("invocation_timestamp"):
+            exec_block["timestamp"] = ctx["invocation_timestamp"]
+
+        # Phases (only if non-default)
+        phases = ctx.get("phases")
+        if phases and phases != ["power"]:
+            exec_block["phases"] = phases
+
+        # Seed (only if set)
+        if ctx.get("seed") is not None:
+            exec_block["seed"] = ctx["seed"]
+
+        # Compression (only if enabled)
+        if ctx.get("compression_enabled"):
+            compression = {"type": ctx.get("compression_type", "none")}
+            if ctx.get("compression_level"):
+                compression["level"] = ctx["compression_level"]
+            exec_block["compression"] = compression
+
+        # Mode (always sql or dataframe)
+        mode_value = None
+        if ctx.get("mode"):
+            mode_value = ctx["mode"]
+        elif isinstance(result.execution_metadata, Mapping):
+            mode_value = result.execution_metadata.get("mode")
+        elif result.platform_info:
+            mode_value = result.platform_info.get("execution_mode")
+        if mode_value:
+            exec_block["mode"] = mode_value
+
+        # Official TPC mode
+        if ctx.get("official"):
+            exec_block["official"] = True
+
+        # Validation mode
+        if ctx.get("validation_mode"):
+            exec_block["validation_mode"] = ctx["validation_mode"]
+
+        # Force flags
+        force_flags = []
+        if ctx.get("force_datagen"):
+            force_flags.append("datagen")
+        if ctx.get("force_upload"):
+            force_flags.append("upload")
+        if force_flags:
+            exec_block["force"] = force_flags
+
+        # Query subset (also in run block, but include here for complete context)
+        if ctx.get("query_subset"):
+            exec_block["query_subset"] = ctx["query_subset"]
+
+        # Plan capture
+        if ctx.get("capture_plans"):
+            exec_block["capture_plans"] = True
+        if ctx.get("strict_plan_capture"):
+            exec_block["strict_plan_capture"] = True
+
+        # Non-interactive
+        if ctx.get("non_interactive"):
+            exec_block["non_interactive"] = True
+
+        # Tuning
+        if ctx.get("tuning_mode"):
+            exec_block["tuning_mode"] = ctx["tuning_mode"]
+
+        if exec_block:
+            payload["execution"] = exec_block
+
+    if "execution" not in payload:
+        exec_block: dict[str, Any] = {}
+        mode_value = None
+        if isinstance(result.execution_metadata, Mapping):
+            mode_value = result.execution_metadata.get("mode")
+        if not mode_value and result.platform_info:
+            mode_value = result.platform_info.get("execution_mode")
+        if mode_value:
+            exec_block["mode"] = mode_value
+        if exec_block:
+            payload["execution"] = exec_block
+
+    return order_dict(payload, CANONICAL_KEY_ORDER)
+
+
+def _shorten_benchmark_name(name: str) -> str:
+    if name.lower().endswith(" benchmark"):
+        return name[: -len(" benchmark")]
+    return name
+
+
+def _build_config_block(result: BenchmarkResults) -> dict[str, Any]:
+    config: dict[str, Any] = {}
+    ctx = result.execution_context or {}
+    run_cfg = {}
+    if isinstance(result.execution_metadata, Mapping):
+        run_cfg = result.execution_metadata.get("run_config") or {}
+
+    compression = None
+    if ctx.get("compression_type") or run_cfg.get("compression"):
+        if ctx.get("compression_type"):
+            compression = {"type": ctx.get("compression_type"), "level": ctx.get("compression_level")}
+        else:
+            compression = run_cfg.get("compression")
+    if compression:
+        config["compression"] = compression
+
+    if ctx.get("seed") is not None:
+        config["seed"] = ctx.get("seed")
+    elif run_cfg.get("seed") is not None:
+        config["seed"] = run_cfg.get("seed")
+
+    if ctx.get("phases"):
+        config["phases"] = ctx.get("phases")
+    elif run_cfg.get("phases"):
+        config["phases"] = run_cfg.get("phases")
+
+    if ctx.get("query_subset"):
+        config["query_subset"] = ctx.get("query_subset")
+    elif run_cfg.get("query_subset"):
+        config["query_subset"] = run_cfg.get("query_subset")
+
+    if run_cfg.get("platform_options"):
+        config["platform_options"] = run_cfg.get("platform_options")
+
+    if run_cfg.get("tuning_mode"):
+        config["tuning_mode"] = run_cfg.get("tuning_mode")
+    if run_cfg.get("tuning_config"):
+        config["tuning_config"] = run_cfg.get("tuning_config")
+
+    # Execution mode and test type for reproducibility
+    exec_mode = None
+    if isinstance(result.execution_metadata, Mapping):
+        exec_mode = result.execution_metadata.get("mode")
+    if not exec_mode and result.platform_info:
+        exec_mode = result.platform_info.get("execution_mode")
+    if exec_mode:
+        config["mode"] = exec_mode
+    # test_type is in benchmark block, not needed here
+
+    return config
+
+
+def _build_phases_block(result: BenchmarkResults) -> dict[str, Any]:
+    phases: dict[str, Any] = {}
+    standard = [
+        "data_generation",
+        "schema_creation",
+        "data_loading",
+        "validation",
+        "power_test",
+        "throughput_test",
+    ]
+
+    setup = getattr(result.execution_phases, "setup", None) if result.execution_phases else None
+    if setup:
+        if setup.data_generation:
+            phases["data_generation"] = {
+                "status": setup.data_generation.status,
+                "duration_ms": setup.data_generation.duration_ms,
+            }
+        if setup.schema_creation:
+            phases["schema_creation"] = {
+                "status": setup.schema_creation.status,
+                "duration_ms": setup.schema_creation.duration_ms,
+            }
+        if setup.data_loading:
+            phases["data_loading"] = {
+                "status": setup.data_loading.status,
+                "duration_ms": setup.data_loading.duration_ms,
+            }
+        if setup.validation:
+            phases["validation"] = {
+                "status": setup.validation.row_count_validation,
+                "duration_ms": setup.validation.duration_ms,
+            }
+
+    if result.execution_phases and result.execution_phases.power_test:
+        phases["power_test"] = {
+            "status": "COMPLETED",
+            "duration_ms": result.execution_phases.power_test.duration_ms,
+        }
+    if result.execution_phases and result.execution_phases.throughput_test:
+        phases["throughput_test"] = {
+            "status": "COMPLETED",
+            "duration_ms": result.execution_phases.throughput_test.duration_ms,
+        }
+
+    if isinstance(result.execution_metadata, Mapping):
+        phase_status = result.execution_metadata.get("phase_status") or {}
+        for name, status in phase_status.items():
+            if name not in phases:
+                phases[name] = status
+
+    for phase in standard:
+        if phase not in phases:
+            phases[phase] = {"status": "NOT_RUN"}
+
+    return phases
 
 
 def build_plans_payload(result: BenchmarkResults) -> dict[str, Any] | None:
@@ -535,10 +808,15 @@ def _build_tpc_metrics(result: BenchmarkResults) -> dict[str, Any] | None:
         metrics["power_at_size"] = result.power_at_size
     if result.throughput_at_size is not None:
         metrics["throughput_at_size"] = result.throughput_at_size
-    if result.qphh_at_size is not None:
-        metrics["qphh_at_size"] = result.qphh_at_size
-    if result.geometric_mean_execution_time is not None:
-        metrics["geometric_mean_ms"] = round(result.geometric_mean_execution_time * 1000, 1)
+    if result.qph_at_size is not None:
+        # Output with benchmark-specific key name based on benchmark_name
+        benchmark_id = normalize_benchmark_id(result.benchmark_name or "")
+        if benchmark_id == "tpcds":
+            metrics["qphds_at_size"] = result.qph_at_size
+        else:
+            metrics["qphh_at_size"] = result.qph_at_size
+    # Note: geometric_mean_ms is in summary.timing (computed from query times),
+    # not here. TPC metrics block contains only TPC-specific metrics.
 
     return metrics if metrics else None
 
@@ -622,7 +900,8 @@ def _build_tables_block(table_statistics: dict[str, Any] | None) -> dict[str, An
         return {}
 
     tables: dict[str, Any] = {}
-    for table_name, stats in table_statistics.items():
+    for table_name in sorted(table_statistics.keys()):
+        stats = table_statistics[table_name]
         if isinstance(stats, dict):
             entry: dict[str, Any] = {}
             if "rows" in stats:
@@ -688,16 +967,36 @@ def _extract_platform_config(platform_info: dict[str, Any]) -> dict[str, Any]:
     if not platform_info:
         return {}
 
-    exclude_keys = {"version", "variant", "name", "platform", "adapter_name", "adapter_version"}
+    exclude_keys = {
+        "version",
+        "variant",
+        "name",
+        "platform",
+        "adapter_name",
+        "adapter_version",
+        "platform_version",
+        "client_library_version",
+        "client_version",
+        "platform_name",
+        "platform_type",
+    }
     config: dict[str, Any] = {}
 
-    for key, value in platform_info.items():
-        if key.lower() not in exclude_keys and value is not None:
+    def extract_recursive(d: dict[str, Any]) -> None:
+        """Recursively extract config, flattening nested 'configuration' keys."""
+        for key, value in d.items():
+            if key.lower() in exclude_keys or value is None:
+                continue
             # Skip empty values
             if isinstance(value, str) and not value:
                 continue
             if isinstance(value, (list, dict)) and not value:
                 continue
-            config[key] = value
+            # Flatten nested configuration
+            if key == "configuration" and isinstance(value, dict):
+                extract_recursive(value)
+            else:
+                config[key] = value
 
+    extract_recursive(platform_info)
     return config

benchbox/core/runner/conversion.py

@@ -24,6 +24,7 @@ from benchbox.utils.format_converters import (
     DeltaConverter,
     IcebergConverter,
     ParquetConverter,
+    VortexConverter,
 )
 
 logger = logging.getLogger(__name__)
@@ -38,6 +39,7 @@ class FormatConversionOrchestrator:
             "parquet": ParquetConverter(),
             "delta": DeltaConverter(),
             "iceberg": IcebergConverter(),
+            "vortex": VortexConverter(),
         }
 
     def convert_benchmark_tables(