benchbox 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

benchbox/utils/format_converters/vortex_converter.py

@@ -0,0 +1,168 @@
+"""Vortex format converter for BenchBox.
+
+Converts TPC benchmark data from TBL (pipe-delimited) format to Vortex columnar format.
+Vortex is a high-performance columnar file format optimized for analytics workloads.
+"""
+
+from __future__ import annotations
+
+import contextlib
+from pathlib import Path
+from typing import Any, Callable
+
+from benchbox.utils.format_converters.base import (
+    BaseFormatConverter,
+    ConversionError,
+    ConversionOptions,
+    ConversionResult,
+    SchemaError,
+)
+
+
+class VortexConverter(BaseFormatConverter):
+    """Converter for TBL -> Vortex format.
+
+    Converts pipe-delimited TPC benchmark data files to Vortex columnar format.
+    Uses the vortex Python package for efficient columnar storage with compression.
+    """
+
+    def get_file_extension(self) -> str:
+        """Get file extension for Vortex format."""
+        return ".vortex"
+
+    def get_format_name(self) -> str:
+        """Get human-readable format name."""
+        return "Vortex"
+
+    def _get_vortex_module(self):
+        """Get the vortex module, raising helpful error if not installed.
+
+        Returns:
+            The vortex module
+
+        Raises:
+            ConversionError: If vortex is not installed
+        """
+        try:
+            import vortex
+
+            return vortex
+        except ImportError as e:
+            raise ConversionError(
+                "Vortex format support requires the 'vortex' package. "
+                "Install it with: uv add vortex --optional table-formats"
+            ) from e
+
+    def convert(
+        self,
+        source_files: list[Path],
+        table_name: str,
+        schema: dict[str, Any],
+        options: ConversionOptions | None = None,
+        progress_callback: Callable[[str, float], None] | None = None,
+    ) -> ConversionResult:
+        """Convert TBL files to Vortex format.
+
+        Args:
+            source_files: List of source TBL file paths (may be sharded)
+            table_name: Name of the table being converted
+            schema: Table schema definition
+            options: Conversion options (uses defaults if None)
+            progress_callback: Optional callback for progress updates
+
+        Returns:
+            ConversionResult with details about the conversion
+
+        Raises:
+            ConversionError: If conversion fails
+            SchemaError: If schema is invalid
+        """
+        # Get vortex module (will raise if not installed)
+        vortex = self._get_vortex_module()
+
+        opts = options or ConversionOptions()
+
+        # Validate inputs
+        self.validate_source_files(source_files)
+        self.validate_schema(schema)
+
+        if progress_callback:
+            progress_callback(f"Starting Vortex conversion for {table_name}", 0.0)
+
+        # Build PyArrow schema
+        try:
+            self._build_arrow_schema(schema)
+        except SchemaError:
+            raise
+        except Exception as e:
+            raise SchemaError(f"Failed to build Arrow schema: {e}") from e
+
+        # Read TBL files using shared method
+        combined_table = self.read_tbl_files(
+            source_files, schema, progress_callback, progress_start=0.0, progress_end=0.8
+        )
+
+        # Get column names for metadata
+        column_names = [col["name"] for col in schema["columns"]]
+
+        # Determine output path
+        source_dir = source_files[0].parent
+        output_dir = opts.output_dir if opts.output_dir else source_dir
+        output_path = output_dir / f"{table_name}.vortex"
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            if progress_callback:
+                progress_callback("Writing Vortex file", 0.9)
+
+            # Convert PyArrow table to Vortex array and write
+            vortex_array = vortex.encoding.array(combined_table)
+            vortex.io.write(vortex_array, output_path)
+
+        except Exception as e:
+            # Clean up partial file if write failed
+            if output_path.exists():
+                with contextlib.suppress(Exception):
+                    output_path.unlink()
+            raise ConversionError(f"Failed to write Vortex file: {e}") from e
+
+        # Calculate metrics
+        source_size = self.calculate_file_size(source_files)
+        output_size = output_path.stat().st_size
+        row_count = combined_table.num_rows
+
+        # Validate row count integrity
+        if opts.validate_row_count:
+            try:
+                self.validate_row_count(source_files, row_count, table_name)
+            except ConversionError:
+                if output_path.exists():
+                    with contextlib.suppress(Exception):
+                        output_path.unlink()
+                raise
+
+        # Build metadata
+        metadata = {
+            "compression": opts.compression,
+            "num_columns": len(column_names),
+        }
+
+        if progress_callback:
+            progress_callback(f"Conversion complete: {row_count:,} rows", 1.0)
+
+        source_format = self._detect_source_format(source_files)
+
+        return ConversionResult(
+            output_files=[output_path],
+            row_count=row_count,
+            source_size_bytes=source_size,
+            output_size_bytes=output_size,
+            metadata=metadata,
+            source_format=source_format,
+            converted_at=self.get_current_timestamp(),
+            conversion_options={
+                "compression": opts.compression,
+                "merge_shards": opts.merge_shards,
+                "validate_row_count": opts.validate_row_count,
+            },
+        )

benchbox/utils/tpc_compilation.py

@@ -103,6 +103,49 @@ def _discover_tpc_paths() -> dict[str, Optional[Path]]:
     return _discovered_paths
 
 
+def get_tpc_templates_dir(benchmark: str) -> Path:
+    """Locate TPC template directory: bundled package first, _sources/ fallback.
+
+    Resolution priority:
+        1. Inside package: benchbox/_binaries/{benchmark}/templates/
+           (works for wheel installs and editable installs)
+        2. _sources/ at repo root: _sources/{benchmark-path}/
+           (development without editable install)
+
+    Args:
+        benchmark: "tpc-h" or "tpc-ds"
+
+    Returns:
+        Path to the templates directory
+
+    Raises:
+        ValueError: If benchmark name is not recognized
+        RuntimeError: If templates directory not found at any location
+    """
+    import benchbox
+
+    package_dir = Path(benchbox.__file__).parent
+
+    # Priority 1: Inside package (_binaries/{benchmark}/templates/)
+    bundled = package_dir / "_binaries" / benchmark / "templates"
+    if bundled.exists():
+        return bundled
+
+    # Priority 2: _sources/ at repo root (development/editable installs)
+    repo_root = package_dir.parent
+    if benchmark == "tpc-h":
+        dev_path = repo_root / "_sources" / "tpc-h" / "dbgen"
+    elif benchmark == "tpc-ds":
+        dev_path = repo_root / "_sources" / "tpc-ds"
+    else:
+        raise ValueError(f"Unknown benchmark: {benchmark}")
+
+    if dev_path.exists():
+        return dev_path
+
+    raise RuntimeError(f"TPC template directory not found for {benchmark}. Checked: {bundled}, {dev_path}")
+
+
 class CompilationStatus(Enum):
     """Status of binary compilation."""
 

benchbox/utils/version.py

@@ -274,12 +274,24 @@ def _gather_version_sources() -> dict[str, Optional[str]]:
 
 
 def check_version_consistency() -> VersionConsistencyResult:
-    """Check if versions are consistent across all sources."""
+    """Check if versions are consistent across all sources.
+
+    In installed mode (wheel), missing external files (pyproject.toml, README.md, etc.)
+    are expected and not treated as inconsistencies. Only actual version mismatches
+    between available sources trigger warnings.
+    """
+    from benchbox.utils.dependencies import is_development_install
 
     versions = _gather_version_sources()
     normalized = {source: _normalize_version(value) for source, value in versions.items()}
 
-    missing_sources = tuple(source for source, value in normalized.items() if value is None)
+    # In installed mode, missing files are expected - only count them if in dev mode
+    is_dev = is_development_install()
+    if is_dev:
+        missing_sources = tuple(source for source, value in normalized.items() if value is None)
+    else:
+        # In installed mode, only require benchbox.__init__ to have a version
+        missing_sources = () if normalized.get("benchbox.__init__") else ("benchbox.__init__",)
 
     # Determine the expected version from the first non-missing entry
     expected_version = next((value for value in normalized.values() if value is not None), None)

{benchbox-0.1.0.dist-info → benchbox-0.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchbox
-Version: 0.1.0
+Version: 0.1.1
 Summary: Embedded benchmark datasets and queries for database evaluation. Implements TPC-H, TPC-DS, TPC-DI and other industry-standard benchmarks.
 Author-email: Joe Harris <joe@benchbox.dev>
 License: MIT
@@ -49,17 +49,16 @@ Requires-Dist: tox>=4.13.0; extra == "dev"
 Requires-Dist: pyspark>=3.5.0; extra == "dev"
 Provides-Extra: docs
 Requires-Dist: sphinx>=7.2.0; extra == "docs"
+Requires-Dist: roman-numerals<4.0; extra == "docs"
 Requires-Dist: furo>=2025.9.25; extra == "docs"
 Requires-Dist: myst-parser>=2.0.0; extra == "docs"
 Requires-Dist: sphinxcontrib-mermaid>=0.8.1; extra == "docs"
 Requires-Dist: pygments>=2.18.0; extra == "docs"
 Requires-Dist: sphinx-tags>=0.4.0; extra == "docs"
 Requires-Dist: sphinx-design>=0.6.0; extra == "docs"
+Requires-Dist: ablog>=0.11.13; python_version >= "3.11" and extra == "docs"
 Provides-Extra: viz
 Requires-Dist: plotly>=5.24.0; extra == "viz"
-Requires-Dist: kaleido>=0.2.1; extra == "viz"
-Requires-Dist: pandas>=2.0.0; extra == "viz"
-Requires-Dist: pillow>=10.0.0; extra == "viz"
 Provides-Extra: all
 Requires-Dist: clickhouse-driver>=0.2.0; extra == "all"
 Requires-Dist: databricks-sql-connector>=2.0.0; extra == "all"
@@ -87,8 +86,6 @@ Requires-Dist: deltalake>=1.2.1; extra == "all"
 Requires-Dist: pyiceberg>=0.10.0; extra == "all"
 Requires-Dist: influxdb3-python>=0.10.0; extra == "all"
 Requires-Dist: plotly>=5.24.0; extra == "all"
-Requires-Dist: kaleido>=0.2.1; extra == "all"
-Requires-Dist: pillow>=10.0.0; extra == "all"
 Requires-Dist: polars>=1.35.2; extra == "all"
 Requires-Dist: dask[distributed]>=2024.1.0; extra == "all"
 Requires-Dist: modin[ray]>=0.32.0; extra == "all"
@@ -226,6 +223,7 @@ Requires-Dist: databricks-connect>=14.0.0; extra == "cloud-spark"
 Requires-Dist: cloudpathlib>=0.15.0; extra == "cloud-spark"
 Provides-Extra: mcp
 Requires-Dist: mcp>=1.2.0; extra == "mcp"
+Requires-Dist: plotly>=5.24.0; extra == "mcp"
 Provides-Extra: spark
 Requires-Dist: pyspark>=4.0.1; extra == "spark"
 Dynamic: license-file
@@ -236,6 +234,7 @@ Dynamic: license-file
 
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Alpha Software](https://img.shields.io/badge/Status-Alpha-orange.svg)](https://github.com/joeharris76/benchbox/issues)
+[![codecov](https://codecov.io/github/joeharris76/BenchBox/graph/badge.svg?token=3NY6DK7MDO)](https://codecov.io/github/joeharris76/BenchBox)
 [![PyPI Release](https://img.shields.io/pypi/v/benchbox)](https://pypi.org/project/benchbox/)
 [![PyPI Downloads](https://img.shields.io/pepy/dt/benchbox.svg?label=PyPI%20Downloads)](https://pypi.org/project/benchbox/)
 
@@ -259,7 +258,7 @@ BenchBox _loosely_ follows [Semantic Versioning](https://semver.org/) using the
 - **MINOR** when we add backward-compatible changes _OR significantly expand functionality_.
 - **PATCH** when we make bug fixes or documentation updates, _bug-fixes may not be backward-compatible_.
 
-Current release: v0.1.0. Check your installation with `benchbox --version`, which also reports metadata consistency diagnostics pulled from `pyproject.toml` and documentation markers.
+Current release: v0.1.1. Check your installation with `benchbox --version`, which also reports metadata consistency diagnostics pulled from `pyproject.toml` and documentation markers.
 
 **For Developers**: See [Release Automation Guide](release/RELEASE_AUTOMATION.md) for the automated release process with reproducible builds and timestamp normalization.
 
@@ -273,8 +272,9 @@ Current release: v0.1.0. Check your installation with `benchbox --version`, whic
 - **Eighteen Benchmarks**: TPC-H, TPC-DS, TPC-DI, TPC-DS-OBT, TPC-H Skew, TPC-Havoc, SSB, AMPLab, JoinOrder, ClickBench, H2ODB, NYC Taxi, TSBS DevOps, CoffeeShop, TPC-H Data Vault, Read Primitives, Write Primitives, Transaction Primitives
 - **Cross-Database**: Same benchmarks work on any database platform
 - **DataFrame Mode**: Native DataFrame API benchmarking with Polars, Pandas, and 6 other libraries
-- **Supported Platforms**: DuckDB, SQLite, DataFusion, PostgreSQL, Polars, Databricks, Snowflake, BigQuery, Redshift, ClickHouse, Trino, Presto, Spark, Athena, Firebolt, Azure Synapse, CUDF (GPU)
-- **DataFrame Platforms**: Polars-DF, Pandas-DF, DataFusion-DF, DuckDB-DF, Modin-DF, Dask-DF, cuDF-DF, PySpark-DF
+- **SQL Platforms** (31): DuckDB, MotherDuck, SQLite, DataFusion, PostgreSQL, TimescaleDB, Polars, ClickHouse, Firebolt, InfluxDB, Databricks, Snowflake, BigQuery, Redshift, Azure Synapse, Microsoft Fabric Warehouse, Trino, Starburst, Presto, Athena, Spark, PySpark, AWS Glue, Amazon EMR Serverless, Athena Spark, GCP Dataproc, GCP Dataproc Serverless, Microsoft Fabric Spark, Azure Synapse Spark, Snowpark Connect, Onehouse Quanton
+- **DataFrame Platforms** (7): Polars-DF, Pandas-DF, DataFusion-DF, Modin-DF, Dask-DF, cuDF-DF (GPU), PySpark-DF
+- **Open Table Formats**: Delta Lake, Apache Iceberg, Apache Hudi (via Databricks, Quanton, Trino, Spark platforms)
 - **SQL Translation**: Automatic query conversion between SQL dialects
 - **Self-Contained Python Package**: Core install requires no external database servers or system dependencies; opt-in to extra package installs for cloud platforms when needed.
 
@@ -311,7 +311,7 @@ BenchBox is one of several open-source database benchmarking tools. Each has dif
 
 BenchBox focuses on OLAP analytic benchmarks while [BenchBase](https://github.com/cmu-db/benchbase) focuses on OLTP transactional benchmarks.
 
-BenchBox provides Python-native benchmarking with embedded data generation, while BenchBase uses Java with JDBC drivers. Both have their setup requirements—BenchBox requires Python dependencies and database connections, while BenchBase requires Java and JDBC setup.
+BenchBox provides Python-native benchmarking with embedded data generation, while BenchBase uses Java with JDBC drivers. Both have their setup requirements - BenchBox requires Python dependencies and database connections, while BenchBase requires Java and JDBC setup.
 
 Consider BenchBox for analytical workloads when you prefer Python-based tooling. Consider BenchBase for transactional workloads or when you need mature, production-tested benchmarking infrastructure with diverse OLTP workloads (TPC-C, Twitter, YCSB, etc.).
 
@@ -319,9 +319,9 @@ Consider BenchBox for analytical workloads when you prefer Python-based tooling.
 
 BenchBox and [HammerDB](https://www.hammerdb.com/) (hosted by the TPC Council) target different workload types.
 
-HammerDB focuses on **OLTP transactional benchmarks** (TPROC-C derived from TPC-C) with support for enterprise databases—Oracle, SQL Server, PostgreSQL, MySQL/MariaDB, and IBM Db2. It uses Tcl and provides GUI, CLI, and web service interfaces. HammerDB measures throughput in NOPM (New Orders Per Minute) and has decades of enterprise credibility.
+HammerDB focuses on **OLTP transactional benchmarks** (TPROC-C derived from TPC-C) with support for enterprise databases - Oracle, SQL Server, PostgreSQL, MySQL/MariaDB, and IBM Db2. It uses Tcl and provides GUI, CLI, and web service interfaces. HammerDB measures throughput in NOPM (New Orders Per Minute) and has decades of enterprise credibility.
 
-BenchBox focuses on **OLAP analytical benchmarks** (TPC-H, TPC-DS, ClickBench, etc.) with support for the broad platform spectrum—from embedded engines like DuckDB and DataFusion, through DataFrame libraries like Polars and Pandas, to cloud data warehouses like Snowflake, BigQuery, and Databricks.
+BenchBox focuses on **OLAP analytical benchmarks** (TPC-H, TPC-DS, ClickBench, etc.) with support for the broad platform spectrum - from embedded engines like DuckDB and DataFusion, through DataFrame libraries like Polars and Pandas, to cloud data warehouses like Snowflake, BigQuery, and Databricks.
 
 Consider HammerDB when testing transactional throughput on enterprise databases or when you need TPC Council-sponsored credibility. Consider BenchBox when benchmarking analytical queries across cloud data warehouses, embedded engines, or DataFrame libraries.
 
@@ -329,9 +329,9 @@ Consider HammerDB when testing transactional throughput on enterprise databases
 
 BenchBox and [LakeBench](https://github.com/mwc360/LakeBench) are both Python-based benchmarking frameworks, but target different ecosystems.
 
-LakeBench focuses on **lakehouse compute engines** (Spark, Fabric, Synapse, HDInsight) and evaluates end-to-end ELT workflows—ingestion, transformation, maintenance, and queries—using Delta Lake tables. It offers 4 benchmarks including ELTBench, a custom workflow-oriented benchmark.
+LakeBench focuses on **lakehouse compute engines** (Spark, Fabric, Synapse, HDInsight) and evaluates end-to-end ELT workflows - ingestion, transformation, maintenance, and queries - using Delta Lake tables. It offers 4 benchmarks including ELTBench, a custom workflow-oriented benchmark.
 
-BenchBox focuses on the **broad ecosystem of analytic platforms**—from single-node engines like DuckDB, to DataFrame libraries like Polars and Pandas, through to cloud data warehouses like Snowflake, BigQuery, and Redshift. It provides 18 benchmarks including TPC standards, academic workloads like SSB and JoinOrder, and BenchBox-original benchmarks like TPC-Havoc for optimizer stress testing.
+BenchBox focuses on the **broad ecosystem of analytic platforms**-from single-node engines like DuckDB, to DataFrame libraries like Polars and Pandas, through to cloud data warehouses like Snowflake, BigQuery, and Redshift. It provides 18 benchmarks including TPC standards, academic workloads like SSB and JoinOrder, and BenchBox-original benchmarks like TPC-Havoc for optimizer stress testing.
 
 Consider LakeBench when evaluating Spark-based lakehouse engines, testing complete ELT pipeline performance, or working primarily in Microsoft Fabric/Azure environments. Consider BenchBox when benchmarking across the analytic platform spectrum, needing benchmark variety beyond TPC standards, or comparing DataFrame libraries alongside SQL engines.
 
@@ -552,7 +552,7 @@ These libraries are required for every installation and provide complete local b
 - **duckdb** – Embedded analytical database engine
 - **pytest libraries** – Testing framework components for built-in validation
 
-The core package includes all necessary Python dependencies for local benchmarking—DuckDB is embedded and ready to go. No external database servers or system installations are required for basic functionality.
+The core package includes all necessary Python dependencies for local benchmarking - DuckDB is embedded and ready to go. No external database servers or system installations are required for basic functionality.
 
 #### Optional Platform Dependencies (installed on demand)