benchbox 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

benchbox/core/schemas.py

@@ -140,7 +140,7 @@ class RunConfig(BaseModel):
     def validate_convert_format(cls, v: Optional[str]) -> Optional[str]:
         """Validate convert_format is supported."""
         if v is not None:
-            allowed = {"parquet", "delta", "iceberg"}
+            allowed = {"parquet", "vortex", "delta", "iceberg"}
             v_lower = v.lower()
             if v_lower not in allowed:
                 raise ValueError(f"convert_format must be one of {allowed}, got: {v}")
@@ -266,7 +266,7 @@ class DatabaseConfig(BaseModel):
     driver_version: Optional[str] = None
     driver_version_resolved: Optional[str] = None
     driver_auto_install: bool = False
-    execution_mode: Optional[Literal["sql", "dataframe"]] = None
+    execution_mode: Optional[Literal["sql", "dataframe", "data_only"]] = None
 
     @field_validator("type")
     @classmethod
@@ -338,7 +338,7 @@ class PlatformInfo(BaseModel):
     enabled: bool
     requirements: list[str]
     installation_command: str
-    recommended: bool = False
+    adoption: str = "niche"  # One of: mainstream, established, emerging, niche
     category: str = "database"
     supports: list[str] = Field(default_factory=list)
     driver_package: Optional[str] = None
@@ -346,6 +346,147 @@ class PlatformInfo(BaseModel):
     driver_version_resolved: Optional[str] = None
 
 
+class ExecutionContext(BaseModel):
+    """Execution context for reproducibility - constructed by all entry points.
+
+    This model captures all execution parameters needed to reproduce a benchmark run.
+    It is constructed at each entry point (CLI, MCP, Python API) and flows through
+    core execution to be embedded in results.
+
+    Example:
+        >>> ctx = ExecutionContext(
+        ...     entry_point="cli",
+        ...     phases=["load", "power"],
+        ...     seed=42,
+        ...     compression_type="zstd",
+        ... )
+        >>> ctx.to_cli_string("duckdb", "tpch", 1.0)
+        'benchbox run --platform duckdb --benchmark tpch --scale 1.0 --phases load,power --seed 42 --compression zstd'
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    # Identity & provenance
+    entry_point: Literal["cli", "mcp", "python_api"] = "python_api"
+    invocation_timestamp: datetime = Field(default_factory=datetime.now)
+
+    # Phases
+    phases: list[str] = Field(default_factory=lambda: ["power"])
+
+    # Determinism
+    seed: Optional[int] = None
+
+    # Compression
+    compression_enabled: bool = False
+    compression_type: str = "none"
+    compression_level: Optional[int] = None
+
+    # Execution mode
+    mode: Literal["sql", "dataframe"] = "sql"
+
+    # TPC compliance
+    official: bool = False
+
+    # Validation
+    validation_mode: Optional[str] = None
+
+    # Force flags
+    force_datagen: bool = False
+    force_upload: bool = False
+
+    # Query subset (also in RunConfig, but included for complete CLI reconstruction)
+    query_subset: Optional[list[str]] = None
+
+    # Plan capture
+    capture_plans: bool = False
+    strict_plan_capture: bool = False
+
+    # Non-interactive
+    non_interactive: bool = False
+
+    # Tuning
+    tuning_mode: Optional[str] = None  # "tuned", "notuning", "auto", or path
+
+    def to_cli_args(self) -> list[str]:
+        """Reconstruct CLI arguments from non-default values.
+
+        Returns:
+            List of CLI arguments like ["--phases", "load,power", "--seed", "42"]
+        """
+        args: list[str] = []
+
+        # Phases (default is ["power"])
+        if self.phases != ["power"]:
+            args.extend(["--phases", ",".join(self.phases)])
+
+        # Seed
+        if self.seed is not None:
+            args.extend(["--seed", str(self.seed)])
+
+        # Compression
+        if self.compression_enabled and self.compression_type != "none":
+            if self.compression_level is not None:
+                args.extend(["--compression", f"{self.compression_type}:{self.compression_level}"])
+            else:
+                args.extend(["--compression", self.compression_type])
+
+        # Mode (default is "sql")
+        if self.mode != "sql":
+            args.extend(["--mode", self.mode])
+
+        # Official TPC mode
+        if self.official:
+            args.append("--official")
+
+        # Validation
+        if self.validation_mode:
+            args.extend(["--validation", self.validation_mode])
+
+        # Force flags
+        force_parts = []
+        if self.force_datagen:
+            force_parts.append("datagen")
+        if self.force_upload:
+            force_parts.append("upload")
+        if force_parts:
+            args.extend(["--force", ",".join(force_parts)])
+
+        # Query subset
+        if self.query_subset:
+            args.extend(["--queries", ",".join(self.query_subset)])
+
+        # Plan capture
+        if self.capture_plans:
+            args.append("--capture-plans")
+        if self.strict_plan_capture:
+            args.extend(["--plan-config", "strict:true"])
+
+        # Non-interactive
+        if self.non_interactive:
+            args.append("--non-interactive")
+
+        # Tuning
+        if self.tuning_mode and self.tuning_mode != "notuning":
+            args.extend(["--tuning", self.tuning_mode])
+
+        return args
+
+    def to_cli_string(self, platform: str, benchmark: str, scale: float) -> str:
+        """Generate complete CLI command string.
+
+        Args:
+            platform: Target platform name
+            benchmark: Benchmark name
+            scale: Scale factor
+
+        Returns:
+            Complete CLI command like "benchbox run --platform duckdb --benchmark tpch --scale 1.0"
+        """
+        base = f"benchbox run --platform {platform} --benchmark {benchmark} --scale {scale}"
+        args = self.to_cli_args()
+        return f"{base} {' '.join(args)}" if args else base
+
+
 class DryRunResult(BaseModel):
     """Contains all information gathered during a dry run."""
 
@@ -375,5 +516,6 @@ __all__ = [
     "SystemProfile",
     "LibraryInfo",
     "PlatformInfo",
+    "ExecutionContext",
     "DryRunResult",
 ]

benchbox/core/ssb/generator.py

@@ -27,6 +27,7 @@ from pathlib import Path
 
 from benchbox.utils.cloud_storage import CloudStorageGeneratorMixin, create_path_handler
 from benchbox.utils.compression_mixin import CompressionMixin
+from benchbox.utils.file_format import detect_compression
 
 
 class SSBDataGenerator(CompressionMixin, CloudStorageGeneratorMixin):
@@ -305,12 +306,13 @@ class SSBDataGenerator(CompressionMixin, CloudStorageGeneratorMixin):
             # Count rows efficiently
             rows = 0
             try:
-                if str(p).endswith(".gz"):
+                compression = detect_compression(p)
+                if compression == "gzip":
                     import gzip
 
                     with gzip.open(p, "rt") as f:
                         rows = sum(1 for _ in f)
-                elif str(p).endswith(".zst"):
+                elif compression == "zstd":
                     try:
                         import zstandard as zstd
 
@@ -321,6 +323,16 @@ class SSBDataGenerator(CompressionMixin, CloudStorageGeneratorMixin):
                             rows = sum(1 for _ in io.TextIOWrapper(reader))
                     except Exception:
                         rows = 0
+                elif compression == "bzip2":
+                    import bz2
+
+                    with bz2.open(p, "rt") as f:
+                        rows = sum(1 for _ in f)
+                elif compression == "xz":
+                    import lzma
+
+                    with lzma.open(p, "rt") as f:
+                        rows = sum(1 for _ in f)
                 else:
                     with open(p, "rb") as f:
                         rows = sum(1 for _ in f)

benchbox/core/tpc_compliance.py

@@ -292,9 +292,9 @@ class TPCOfficialMetrics:
         power_at_size = (3600.0 * self.scale_factor) / power_time
         throughput_at_size = (num_streams * 3600.0 * self.scale_factor) / throughput_time
 
-        import math
+        from benchbox.core.results.metrics import TPCMetricsCalculator
 
-        return math.sqrt(power_at_size * throughput_at_size)
+        return TPCMetricsCalculator.calculate_qph(power_at_size, throughput_at_size)
 
     def calculate_qphds_size(self, power_time: float, throughput_time: float, num_streams: int) -> float:
         """Calculate QphDS@Size metric for TPC-DS."""
@@ -304,9 +304,9 @@ class TPCOfficialMetrics:
         power_at_size = (3600.0 * self.scale_factor) / power_time
         throughput_at_size = (num_streams * 3600.0 * self.scale_factor) / throughput_time
 
-        import math
+        from benchbox.core.results.metrics import TPCMetricsCalculator
 
-        return math.sqrt(power_at_size * throughput_at_size)
+        return TPCMetricsCalculator.calculate_qph(power_at_size, throughput_at_size)
 
 
 class TPCTimer:

benchbox/core/tpc_metrics.py

@@ -4,7 +4,7 @@ This module implements the official TPC metrics calculation engine that computes
 standard TPC metrics according to the official TPC specifications.
 
 Key metrics implemented:
-- Power@Size: 3600 × Scale_Factor / Power_Test_Time
+- Power@Size: 3600 × Scale_Factor / geometric_mean(query_times)
 - Throughput@Size: Num_Streams × 3600 × Scale_Factor / Throughput_Test_Time
 - QphH@Size: sqrt(Power@Size × Throughput@Size) [TPC-H]
 - QphDS@Size: sqrt(Power@Size × Throughput@Size) [TPC-DS]
@@ -240,7 +240,7 @@ class PowerMetrics:
         """
         Calculate Power@Size metric.
 
-        Formula: Power@Size = 3600 × Scale_Factor / Power_Test_Time
+        Formula: Power@Size = 3600 × Scale_Factor / geometric_mean(query_times)
 
         Args:
             results: PowerTestResults containing test execution data
@@ -253,8 +253,13 @@ class PowerMetrics:
         if validation_errors:
             return 0.0, validation_errors
 
-        # Calculate Power@Size
-        power_at_size = (3600.0 * results.scale_factor) / results.total_time
+        exec_times = [r.execution_time for r in results.test_results if r.success and r.execution_time > 0]
+        if not exec_times:
+            validation_errors.append("No valid query execution times for Power@Size calculation")
+            return 0.0, validation_errors
+
+        geometric_mean = statistics.geometric_mean(exec_times)
+        power_at_size = (3600.0 * results.scale_factor) / geometric_mean
 
         # Additional validation on result
         if power_at_size <= 0:

benchbox/core/tpcdi/generator/manifest.py

@@ -5,6 +5,8 @@ from __future__ import annotations
 import json
 from pathlib import Path
 
+from benchbox.utils.file_format import detect_compression
+
 
 class ManifestMixin:
     """Provide manifest persistence and validation helpers."""
@@ -49,12 +51,13 @@ class ManifestMixin:
             size = p.stat().st_size if p.exists() else 0
             rows = 0
             try:
-                if str(p).endswith(".gz"):
+                compression = detect_compression(p)
+                if compression == "gzip":
                     import gzip
 
                     with gzip.open(p, "rt") as f:
                         rows = sum(1 for _ in f)
-                elif str(p).endswith(".zst"):
+                elif compression == "zstd":
                     try:
                         import zstandard as zstd
 
@@ -65,6 +68,16 @@ class ManifestMixin:
                             rows = sum(1 for _ in io.TextIOWrapper(reader))
                     except Exception:
                         rows = 0
+                elif compression == "bzip2":
+                    import bz2
+
+                    with bz2.open(p, "rt") as f:
+                        rows = sum(1 for _ in f)
+                elif compression == "xz":
+                    import lzma
+
+                    with lzma.open(p, "rt") as f:
+                        rows = sum(1 for _ in f)
                 else:
                     with open(p, "rb") as f:
                         rows = sum(1 for _ in f)

benchbox/core/tpcds/benchmark/runner.py

@@ -18,6 +18,7 @@ from benchbox.core.validation import (
     DataValidationEngine,
     ValidationResult,
 )
+from benchbox.utils.file_format import get_delimiter_for_file
 
 from ..c_tools import TPCDSCTools
 from ..generator import TPCDSDataGenerator
@@ -1012,13 +1013,8 @@ class TPCDSBenchmark(BaseBenchmark):
 
         rows_loaded = 0
 
-        # Determine the file format and delimiter
-        if data_file.suffix == ".csv":
-            # CSV format (converted from .dat)
-            delimiter = ","
-        else:
-            # Original .dat format with pipe delimiter
-            delimiter = "|"
+        # Determine the delimiter based on file format (handles compression)
+        delimiter = get_delimiter_for_file(data_file)
 
         # Read and process the delimited file
         with open(data_file, encoding="utf-8") as f:

benchbox/core/tpcds/c_tools.py

@@ -39,34 +39,33 @@ def _resolve_tpcds_tool_and_template_paths() -> tuple[Path, Path]:
 
         To eliminate maintenance burden and prevent template divergence, we:
         1. Use platform-specific binaries from _binaries/tpc-ds/{platform}/
-        2. Use centralized templates from _sources/tpc-ds/query_templates/
+        2. Use centralized templates from _binaries/tpc-ds/templates/query_templates/
+           (bundled inside the package for wheel installs)
 
         This approach is safe because:
         - dsqgen uses the DSS_QUERY environment variable to locate templates
         - Templates and binaries don't need to be co-located
         - Data files (tpcds.idx, tpcds.dst) are copied from binary location at runtime
-
-        Template duplication in _binaries/ directories is legacy and unused.
     """
+    from benchbox.utils.tpc_compilation import get_tpc_templates_dir
 
     compiler = get_tpc_compiler(auto_compile=False)
 
-    import benchbox
-
-    repo_root = Path(benchbox.__file__).parent.parent
-
-    # ALWAYS use centralized templates from _sources/ (single source of truth)
-    templates_path = repo_root / "_sources/tpc-ds/query_templates"
+    # Resolve templates: bundled package first, then _sources/ fallback
+    templates_root = get_tpc_templates_dir("tpc-ds")
+    templates_path = templates_root / "query_templates"
 
     # Use platform-specific binaries from precompiled bundle when available
     if compiler.precompiled_base:
         platform_str = compiler._get_platform_string()
         bundle_root = compiler.precompiled_base / "tpc-ds" / platform_str
         if bundle_root.exists():
-            # Return: platform binaries + centralized templates
             return bundle_root, templates_path
 
     # Fallback to source tools if no precompiled binaries
+    import benchbox
+
+    repo_root = Path(benchbox.__file__).parent.parent
     tools_path = repo_root / "_sources/tpc-ds/tools"
     return tools_path, templates_path
 
@@ -256,9 +255,6 @@ class DSQGenBinary:
         """Generate query using dsqgen binary with minimal wrapper."""
         cmd = [str(self.dsqgen_path)]
 
-        # Essential dsqgen parameters
-        templates_list = self.templates_dir / "templates.lst"
-
         # Template and dialect
         # Multi-part queries (14, 23, 24, 39) have two SQL statements in a single template
         # For these, we use the base template and extract the appropriate part later
@@ -284,21 +280,15 @@ class DSQGenBinary:
             # Fall back to main templates relative path on any error
             template_rel = f"query_templates/{template_name}"
 
-        cmd.extend(["-INPUT", str(templates_list)])
-
-        # For dialect files, dsqgen looks for them in the -DIRECTORY path
-        # Dialect templates (like netezza.tpl) are in query_templates/, so always use that as DIRECTORY
-        cmd.extend(["-DIRECTORY", str(self.templates_dir)])
-
         if "query_variants/" in template_rel:
             # For variant templates, use relative path from query_templates (../ to access sibling directory)
             # query_variants/query14a.tpl becomes ../query_variants/query14a.tpl
-            cmd.extend(["-TEMPLATE", f"../{template_rel}"])
+            template_arg = f"../{template_rel}"
         else:
             # For main templates, remove the query_templates/ prefix since we're already in that directory
-            template_name_only = template_rel.replace("query_templates/", "")
-            cmd.extend(["-TEMPLATE", template_name_only])
+            template_arg = template_rel.replace("query_templates/", "")
 
+        cmd.extend(["-TEMPLATE", template_arg])
         cmd.extend(["-DIALECT", dialect])
 
         # Scale and generation parameters
@@ -318,11 +308,27 @@ class DSQGenBinary:
             import tempfile
 
             with tempfile.TemporaryDirectory() as temp_dir:
+                temp_path = Path(temp_dir)
+
+                # Symlink templates into temp dir with short names to avoid
+                # exceeding dsqgen's internal 80-char path buffer for -DIRECTORY.
+                # Use single-char name for the templates dir (passed as -DIRECTORY).
+                tpl_dir = temp_path / "q"
+                tpl_dir.symlink_to(self.templates_dir)
+                # Symlink query_variants as sibling so ../query_variants/ resolves
+                var_src = self.templates_dir.parent / "query_variants"
+                if var_src.exists():
+                    (temp_path / "query_variants").symlink_to(var_src)
+
+                # Set -INPUT and -DIRECTORY using short symlinked paths
+                cmd.extend(["-INPUT", str(tpl_dir / "templates.lst")])
+                cmd.extend(["-DIRECTORY", str(tpl_dir)])
+
                 # Set up environment for dsqgen
                 env = os.environ.copy()
-                # Point DSS_QUERY at the TPC-DS source root so dsqgen can find
-                # both query_templates and query_variants directories
-                env["DSS_QUERY"] = str(self.templates_dir.parent)
+                # Point DSS_QUERY at the temp dir which contains both
+                # the q/ (query_templates) and query_variants/ symlinks
+                env["DSS_QUERY"] = str(temp_path)
 
                 # Copy required distribution files to temp directory
                 dsqgen_dir = Path(self.dsqgen_path).parent
@@ -330,11 +336,11 @@ class DSQGenBinary:
                 for dist_file in dist_files:
                     dist_src = dsqgen_dir / dist_file
                     if dist_src.exists():
-                        shutil.copy2(dist_src, Path(temp_dir) / dist_file)
+                        shutil.copy2(dist_src, temp_path / dist_file)
                     # Also check if file exists in tools_dir (source version)
                     dist_src_alt = self.tools_dir / dist_file
-                    if not (Path(temp_dir) / dist_file).exists() and dist_src_alt.exists():
-                        shutil.copy2(dist_src_alt, Path(temp_dir) / dist_file)
+                    if not (temp_path / dist_file).exists() and dist_src_alt.exists():
+                        shutil.copy2(dist_src_alt, temp_path / dist_file)
 
                 result = subprocess.run(
                     cmd,

benchbox/core/tpcds/dataframe_queries/queries.py

@@ -9255,19 +9255,29 @@ def q16_expression_impl(ctx: DataFrameContext) -> Any:
     returned_orders = catalog_returns.select(["cr_order_number"]).unique()
 
     # Find orders shipped from multiple warehouses
-    # First, identify order numbers that have different warehouses
-    order_warehouses = catalog_sales.select(["cs_order_number", "cs_warehouse_sk"]).unique()
-
-    # Self-join to find orders with different warehouses
-    multi_warehouse_orders = (
-        order_warehouses.join(
-            order_warehouses.rename({"cs_warehouse_sk": "cs_warehouse_sk_2"}),
-            on="cs_order_number",
+    # Platform-specific optimization: Polars is faster with n_unique,
+    # DataFusion is faster with self-join pattern
+    platform = getattr(ctx, "platform", "polars")
+    if platform == "datafusion":
+        # DataFusion: self-join pattern is 5x faster
+        order_warehouses = catalog_sales.select(["cs_order_number", "cs_warehouse_sk"]).unique()
+        multi_warehouse_orders = (
+            order_warehouses.join(
+                order_warehouses.rename({"cs_warehouse_sk": "cs_warehouse_sk_2"}),
+                on="cs_order_number",
+            )
+            .filter(col("cs_warehouse_sk") != col("cs_warehouse_sk_2"))
+            .select(["cs_order_number"])
+            .unique()
+        )
+    else:
+        # Polars/others: n_unique pattern is 7x faster
+        multi_warehouse_orders = (
+            catalog_sales.group_by("cs_order_number")
+            .agg(col("cs_warehouse_sk").n_unique().alias("num_warehouses"))
+            .filter(col("num_warehouses") > lit(1))
+            .select("cs_order_number")
         )
-        .filter(col("cs_warehouse_sk") != col("cs_warehouse_sk_2"))
-        .select(["cs_order_number"])
-        .unique()
-    )
 
     # Main query: catalog_sales joined with dimensions
     result = (
@@ -11774,16 +11784,29 @@ def q95_expression_impl(ctx: DataFrameContext) -> Any:
     ws_filtered = web_site.filter(col("web_company_name") == lit("pri"))
 
     # Find orders shipped from multiple warehouses (CTE ws_wh)
-    order_warehouses = web_sales.select(["ws_order_number", "ws_warehouse_sk"]).unique()
-    multi_warehouse_orders = (
-        order_warehouses.join(
-            order_warehouses.rename({"ws_warehouse_sk": "ws_warehouse_sk_2"}),
-            on="ws_order_number",
+    # Platform-specific optimization: Polars is faster with n_unique,
+    # DataFusion is faster with self-join pattern
+    platform = getattr(ctx, "platform", "polars")
+    if platform == "datafusion":
+        # DataFusion: self-join pattern is 5x faster
+        order_warehouses = web_sales.select(["ws_order_number", "ws_warehouse_sk"]).unique()
+        multi_warehouse_orders = (
+            order_warehouses.join(
+                order_warehouses.rename({"ws_warehouse_sk": "ws_warehouse_sk_2"}),
+                on="ws_order_number",
+            )
+            .filter(col("ws_warehouse_sk") != col("ws_warehouse_sk_2"))
+            .select(["ws_order_number"])
+            .unique()
+        )
+    else:
+        # Polars/others: n_unique pattern is 7x faster
+        multi_warehouse_orders = (
+            web_sales.group_by("ws_order_number")
+            .agg(col("ws_warehouse_sk").n_unique().alias("num_warehouses"))
+            .filter(col("num_warehouses") > lit(1))
+            .select("ws_order_number")
         )
-        .filter(col("ws_warehouse_sk") != col("ws_warehouse_sk_2"))
-        .select(["ws_order_number"])
-        .unique()
-    )
 
     # Get returned orders that are also multi-warehouse
     returned_orders = (

benchbox/core/tpcds/generator/filesystem.py

@@ -6,6 +6,7 @@ import json
 import shutil
 from pathlib import Path
 
+from benchbox.utils.file_format import detect_compression, strip_compression_suffix
 from benchbox.utils.scale_factor import format_scale_factor
 
 
@@ -69,12 +70,8 @@ class FileArtifactMixin:
                 valid_parallel_files: list[Path] = []
                 for pf in candidate_files:
                     name = pf.name
-                    if name.endswith(".gz"):
-                        name_core = name[:-3]
-                    elif name.endswith(".zst"):
-                        name_core = name[:-4]
-                    else:
-                        name_core = name
+                    # Strip compression suffix to get the core filename
+                    name_core = strip_compression_suffix(Path(name)).name
                     stem = Path(name_core).stem
                     if stem.startswith(f"{table_name}_"):
                         suffix = stem[len(f"{table_name}_") :]
@@ -131,12 +128,16 @@ class FileArtifactMixin:
         file_size = file_path.stat().st_size
 
         # For compressed files, minimum valid size should be significantly larger than just compressed headers
-        if str(file_path).endswith(".zst"):
+        compression = detect_compression(file_path)
+        if compression == "zstd":
             # Empty zstd files are exactly 9 bytes; consider any value >9 as non-empty
             return file_size > 9
-        elif str(file_path).endswith(".gz"):
+        elif compression == "gzip":
             # Empty gzip files are around ~20 bytes; accept >20 as non-empty
             return file_size > 20
+        elif compression is not None:
+            # For other compression types, use a reasonable minimum
+            return file_size > 20
         else:
             # For uncompressed files, any non-empty file is valid
             return file_size > 0
@@ -295,17 +296,28 @@ class FileArtifactMixin:
                 # Count rows in the file (handles both compressed and uncompressed)
                 row_count = 0
                 try:
-                    # Handle compressed files
-                    if str(file_path).endswith(".gz"):
+                    # Handle compressed files using centralized detection
+                    compression = detect_compression(file_path)
+                    if compression == "gzip":
                         import gzip
 
                         with gzip.open(file_path, "rb") as f:
                             row_count = sum(1 for _ in f)
-                    elif str(file_path).endswith(".zst"):
+                    elif compression == "zstd":
                         import zstandard as zstd
 
                         with zstd.open(file_path, "rb") as f:
                             row_count = sum(1 for _ in f)
+                    elif compression == "bzip2":
+                        import bz2
+
+                        with bz2.open(file_path, "rb") as f:
+                            row_count = sum(1 for _ in f)
+                    elif compression == "xz":
+                        import lzma
+
+                        with lzma.open(file_path, "rb") as f:
+                            row_count = sum(1 for _ in f)
                     else:
                         # Uncompressed file
                         with open(file_path, "rb") as f:

benchbox/core/tpcds/generator/manager.py

@@ -10,6 +10,7 @@ from pathlib import Path
 from benchbox.utils.cloud_storage import CloudStorageGeneratorMixin, create_path_handler
 from benchbox.utils.compression_mixin import CompressionMixin
 from benchbox.utils.data_validation import BenchmarkDataValidator
+from benchbox.utils.file_format import is_tpc_format
 
 from .filesystem import FileArtifactMixin
 from .runner import DsdgenRunnerMixin
@@ -266,10 +267,10 @@ class TPCDSDataGenerator(
                 print(f"⚡ Using bundled TPC-DS sample dataset for scale factor {self.scale_factor}")
             self._copy_sample_dataset(sample_dir, target_dir)
 
-            # If compression is enabled, compress the copied .dat files
+            # If compression is enabled, compress the copied TPC data files
             if self.should_use_compression():
                 for dat_file in list(target_dir.glob("*.dat")):
-                    if dat_file.suffix == ".dat":  # Only compress .dat files, skip .dst and .idx
+                    if is_tpc_format(dat_file):  # Only compress TPC data files, skip .dst and .idx
                         try:
                             self.compress_existing_file(dat_file, remove_original=True)
                         except Exception: