benchbox 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

benchbox/core/tpcds/maintenance_test.py

@@ -505,6 +505,287 @@ class TPCDSMaintenanceTest:
             ],
         }
 
+    def get_maintenance_operations_sql(
+        self,
+        placeholder: str = "?",
+    ) -> dict[str, list[str]]:
+        """Generate maintenance operation SQL for dry-run preview.
+
+        Uses the SAME data generation and SQL construction logic as actual execution
+        via MaintenanceOperations, ensuring dry-run output matches what would actually
+        be executed. This follows the same pattern as TPC-H's get_maintenance_operations_sql().
+
+        Args:
+            placeholder: SQL parameter placeholder style ('?' for DuckDB/SQLite, '%s' for PostgreSQL)
+
+        Returns:
+            Dict mapping operation IDs to lists of SQL statements
+        """
+        result: dict[str, list[str]] = {}
+
+        # Calculate operation counts based on scale factor (same as actual execution)
+        # These match the row counts used in _execute_maintenance_operation
+        insert_rows = max(10, int(self.scale_factor * 100))
+        update_rows = max(5, int(self.scale_factor * 50))
+        delete_rows = max(2, int(self.scale_factor * 20))
+
+        # Initialize maintenance_ops with mock dimension ranges for dry-run
+        # (actual execution would query these from the database)
+        self._init_maintenance_ops_for_dryrun()
+
+        # DM1: INSERT operations for fact tables - uses SAME row generation as actual execution
+        result["DM1_INSERT"] = self._generate_insert_sql_via_maintenance_ops(placeholder, insert_rows)
+
+        # DM2: INSERT operations for returns tables
+        # Note: In actual execution, returns require querying existing sales for FK refs.
+        # For dry-run, we show the SQL structure that would be generated.
+        result["DM2_INSERT_RETURNS"] = self._generate_returns_insert_sql_via_maintenance_ops(
+            placeholder, insert_rows // 10
+        )
+
+        # DM3: UPDATE operations for dimension tables
+        result["DM3_UPDATE"] = self._generate_update_sql_via_maintenance_ops(placeholder, update_rows)
+
+        # DM4: DELETE operations for old data cleanup
+        result["DM4_DELETE"] = self._generate_delete_sql_via_maintenance_ops(placeholder, delete_rows)
+
+        return result
+
+    def _init_maintenance_ops_for_dryrun(self) -> None:
+        """Initialize MaintenanceOperations with mock dimension ranges for dry-run.
+
+        Sets up dimension key ranges based on scale factor estimates, allowing
+        row generation to work without database queries.
+        """
+        # Set up dimension ranges based on TPC-DS scale factor estimates
+        # These are approximate but match the ranges used in actual data generation
+        sf = self.scale_factor
+
+        self.maintenance_ops.dimension_ranges = {
+            "date_dim": (2450815, 2453005),  # Standard TPC-DS date range
+            "time_dim": (0, 86399),  # Seconds in a day
+            "item": (1, max(1, int(18000 * sf))),
+            "customer": (1, max(1, int(100000 * sf))),
+            "customer_demographics": (1, 1920800),  # Fixed cardinality
+            "household_demographics": (1, 7200),  # Fixed cardinality
+            "customer_address": (1, max(1, int(50000 * sf))),
+            "store": (1, max(1, int(12 * sf))),
+            "promotion": (1, max(1, int(300 * sf))),
+            "call_center": (1, max(1, int(6 * sf))),
+            "catalog_page": (1, max(1, int(11718 * sf))),
+            "ship_mode": (1, 20),  # Fixed cardinality
+            "warehouse": (1, max(1, int(5 * sf))),
+            "web_site": (1, max(1, int(30 * sf))),
+            "web_page": (1, max(1, int(60 * sf))),
+        }
+
+        # Seed random generator for reproducible dry-run output
+        self.maintenance_ops.random_gen.seed(42)
+
+    def _generate_insert_sql_via_maintenance_ops(self, placeholder: str, num_rows: int) -> list[str]:
+        """Generate INSERT SQL using the SAME row generation as actual execution.
+
+        This uses MaintenanceOperations._generate_*_row() methods - the exact same
+        methods used during actual maintenance test execution.
+        """
+        statements = []
+        batch_size = min(100, num_rows)  # Same batch size as _execute_batched_insert
+
+        # Store Sales INSERT - using _generate_store_sales_row() (same as _insert_store_sales)
+        store_sales_rows = [self.maintenance_ops._generate_store_sales_row() for _ in range(batch_size)]
+        columns = """SS_SOLD_DATE_SK, SS_SOLD_TIME_SK, SS_ITEM_SK, SS_CUSTOMER_SK,
+            SS_CDEMO_SK, SS_HDEMO_SK, SS_ADDR_SK, SS_STORE_SK, SS_PROMO_SK,
+            SS_TICKET_NUMBER, SS_QUANTITY, SS_WHOLESALE_COST, SS_LIST_PRICE,
+            SS_SALES_PRICE, SS_EXT_DISCOUNT_AMT, SS_EXT_SALES_PRICE,
+            SS_EXT_WHOLESALE_COST, SS_EXT_LIST_PRICE, SS_EXT_TAX,
+            SS_COUPON_AMT, SS_NET_PAID, SS_NET_PAID_INC_TAX, SS_NET_PROFIT"""
+
+        # Build multi-row VALUES clause (same as _execute_batched_insert)
+        row_placeholders = ", ".join([placeholder] * 23)
+        values_placeholders = ", ".join([f"({row_placeholders})" for _ in store_sales_rows])
+        sample_row = store_sales_rows[0] if store_sales_rows else ()
+
+        statements.append(
+            f"-- TPC-DS Maintenance: Insert {num_rows} new store sales records\n"
+            f"-- Sample row values: {sample_row[:5]}...\n"
+            f"INSERT INTO STORE_SALES ({columns}) VALUES {values_placeholders}"
+        )
+
+        # Catalog Sales INSERT - using _generate_catalog_sales_row() (same as _insert_catalog_sales)
+        catalog_sales_rows = [self.maintenance_ops._generate_catalog_sales_row() for _ in range(batch_size)]
+        columns = """CS_SOLD_DATE_SK, CS_SOLD_TIME_SK, CS_SHIP_DATE_SK, CS_BILL_CUSTOMER_SK,
+            CS_BILL_CDEMO_SK, CS_BILL_HDEMO_SK, CS_BILL_ADDR_SK, CS_SHIP_CUSTOMER_SK,
+            CS_SHIP_CDEMO_SK, CS_SHIP_HDEMO_SK, CS_SHIP_ADDR_SK, CS_CALL_CENTER_SK,
+            CS_CATALOG_PAGE_SK, CS_SHIP_MODE_SK, CS_WAREHOUSE_SK, CS_ITEM_SK,
+            CS_PROMO_SK, CS_ORDER_NUMBER, CS_QUANTITY, CS_WHOLESALE_COST,
+            CS_LIST_PRICE, CS_SALES_PRICE, CS_EXT_DISCOUNT_AMT, CS_EXT_SALES_PRICE,
+            CS_EXT_WHOLESALE_COST, CS_EXT_LIST_PRICE, CS_EXT_TAX, CS_COUPON_AMT,
+            CS_EXT_SHIP_COST, CS_NET_PAID, CS_NET_PAID_INC_TAX, CS_NET_PAID_INC_SHIP,
+            CS_NET_PAID_INC_SHIP_TAX, CS_NET_PROFIT"""
+
+        row_placeholders = ", ".join([placeholder] * 34)
+        values_placeholders = ", ".join([f"({row_placeholders})" for _ in catalog_sales_rows])
+        sample_row = catalog_sales_rows[0] if catalog_sales_rows else ()
+
+        statements.append(
+            f"-- TPC-DS Maintenance: Insert {num_rows} new catalog sales records\n"
+            f"-- Sample row values: {sample_row[:5]}...\n"
+            f"INSERT INTO CATALOG_SALES ({columns}) VALUES {values_placeholders}"
+        )
+
+        # Web Sales INSERT - using _generate_web_sales_row() (same as _insert_web_sales)
+        web_sales_rows = [self.maintenance_ops._generate_web_sales_row() for _ in range(batch_size)]
+        columns = """WS_SOLD_DATE_SK, WS_SOLD_TIME_SK, WS_SHIP_DATE_SK, WS_ITEM_SK,
+            WS_BILL_CUSTOMER_SK, WS_BILL_CDEMO_SK, WS_BILL_HDEMO_SK, WS_BILL_ADDR_SK,
+            WS_SHIP_CUSTOMER_SK, WS_SHIP_CDEMO_SK, WS_SHIP_HDEMO_SK, WS_SHIP_ADDR_SK,
+            WS_WEB_PAGE_SK, WS_WEB_SITE_SK, WS_SHIP_MODE_SK, WS_WAREHOUSE_SK,
+            WS_PROMO_SK, WS_ORDER_NUMBER, WS_QUANTITY, WS_WHOLESALE_COST,
+            WS_LIST_PRICE, WS_SALES_PRICE, WS_EXT_DISCOUNT_AMT, WS_EXT_SALES_PRICE,
+            WS_EXT_WHOLESALE_COST, WS_EXT_LIST_PRICE, WS_EXT_TAX, WS_COUPON_AMT,
+            WS_EXT_SHIP_COST, WS_NET_PAID, WS_NET_PAID_INC_TAX, WS_NET_PAID_INC_SHIP,
+            WS_NET_PAID_INC_SHIP_TAX, WS_NET_PROFIT"""
+
+        row_placeholders = ", ".join([placeholder] * 34)
+        values_placeholders = ", ".join([f"({row_placeholders})" for _ in web_sales_rows])
+        sample_row = web_sales_rows[0] if web_sales_rows else ()
+
+        statements.append(
+            f"-- TPC-DS Maintenance: Insert {num_rows} new web sales records\n"
+            f"-- Sample row values: {sample_row[:5]}...\n"
+            f"INSERT INTO WEB_SALES ({columns}) VALUES {values_placeholders}"
+        )
+
+        return statements
+
+    def _generate_returns_insert_sql_via_maintenance_ops(self, placeholder: str, num_rows: int) -> list[str]:
+        """Generate returns INSERT SQL showing the structure used in actual execution.
+
+        Note: In actual execution, returns require querying existing sales records
+        to get valid FK references (ticket_number, item_sk, etc.). For dry-run,
+        we show the SQL structure with placeholder references.
+        """
+        statements = []
+
+        # Store Returns - actual execution queries STORE_SALES first, then calls
+        # _generate_store_returns_from_sale() with the sale record
+        columns = """SR_RETURNED_DATE_SK, SR_RETURN_TIME_SK, SR_ITEM_SK, SR_CUSTOMER_SK,
+            SR_CDEMO_SK, SR_HDEMO_SK, SR_ADDR_SK, SR_STORE_SK, SR_REASON_SK,
+            SR_TICKET_NUMBER, SR_RETURN_QUANTITY, SR_RETURN_AMT, SR_RETURN_TAX,
+            SR_RETURN_AMT_INC_TAX, SR_FEE, SR_RETURN_SHIP_COST, SR_REFUNDED_CASH,
+            SR_REVERSED_CHARGE, SR_STORE_CREDIT, SR_NET_LOSS"""
+
+        row_placeholders = ", ".join([placeholder] * 20)
+        statements.append(
+            f"-- TPC-DS Maintenance: Insert {num_rows} store returns\n"
+            f"-- Step 1: Query STORE_SALES to get valid (ticket_number, item_sk) pairs\n"
+            f"-- Step 2: Generate returns referencing those sales\n"
+            f"INSERT INTO STORE_RETURNS ({columns}) VALUES ({row_placeholders})"
+        )
+
+        # Catalog Returns - actual execution queries CATALOG_SALES first
+        columns = """CR_RETURNED_DATE_SK, CR_RETURNED_TIME_SK, CR_ITEM_SK, CR_REFUNDED_CUSTOMER_SK,
+            CR_REFUNDED_CDEMO_SK, CR_REFUNDED_HDEMO_SK, CR_REFUNDED_ADDR_SK,
+            CR_RETURNING_CUSTOMER_SK, CR_RETURNING_CDEMO_SK, CR_RETURNING_HDEMO_SK,
+            CR_RETURNING_ADDR_SK, CR_CALL_CENTER_SK, CR_CATALOG_PAGE_SK, CR_SHIP_MODE_SK,
+            CR_WAREHOUSE_SK, CR_REASON_SK, CR_ORDER_NUMBER, CR_RETURN_QUANTITY,
+            CR_RETURN_AMOUNT, CR_RETURN_TAX, CR_RETURN_AMT_INC_TAX, CR_FEE,
+            CR_RETURN_SHIP_COST, CR_REFUNDED_CASH, CR_REVERSED_CHARGE, CR_STORE_CREDIT,
+            CR_NET_LOSS"""
+
+        row_placeholders = ", ".join([placeholder] * 27)
+        statements.append(
+            f"-- TPC-DS Maintenance: Insert {num_rows} catalog returns\n"
+            f"-- Step 1: Query CATALOG_SALES to get valid (order_number, item_sk) pairs\n"
+            f"-- Step 2: Generate returns referencing those sales\n"
+            f"INSERT INTO CATALOG_RETURNS ({columns}) VALUES ({row_placeholders})"
+        )
+
+        # Web Returns - actual execution queries WEB_SALES first
+        columns = """WR_RETURNED_DATE_SK, WR_RETURNED_TIME_SK, WR_ITEM_SK, WR_REFUNDED_CUSTOMER_SK,
+            WR_REFUNDED_CDEMO_SK, WR_REFUNDED_HDEMO_SK, WR_REFUNDED_ADDR_SK,
+            WR_RETURNING_CUSTOMER_SK, WR_RETURNING_CDEMO_SK, WR_RETURNING_HDEMO_SK,
+            WR_RETURNING_ADDR_SK, WR_WEB_PAGE_SK, WR_REASON_SK, WR_ORDER_NUMBER,
+            WR_RETURN_QUANTITY, WR_RETURN_AMT, WR_RETURN_TAX, WR_RETURN_AMT_INC_TAX,
+            WR_FEE, WR_RETURN_SHIP_COST, WR_REFUNDED_CASH, WR_REVERSED_CHARGE,
+            WR_ACCOUNT_CREDIT, WR_NET_LOSS"""
+
+        row_placeholders = ", ".join([placeholder] * 24)
+        statements.append(
+            f"-- TPC-DS Maintenance: Insert {num_rows} web returns\n"
+            f"-- Step 1: Query WEB_SALES to get valid (order_number, item_sk) pairs\n"
+            f"-- Step 2: Generate returns referencing those sales\n"
+            f"INSERT INTO WEB_RETURNS ({columns}) VALUES ({row_placeholders})"
+        )
+
+        return statements
+
+    def _generate_update_sql_via_maintenance_ops(self, placeholder: str, num_rows: int) -> list[str]:
+        """Generate UPDATE SQL using the SAME logic as actual execution.
+
+        These match the UPDATE statements in MaintenanceOperations._update_* methods.
+        """
+        statements = []
+
+        # Customer UPDATE - matches _update_customer logic
+        statements.append(
+            f"-- TPC-DS Maintenance: Update {num_rows} customer records\n"
+            f"-- Same logic as MaintenanceOperations._update_customer()\n"
+            f"UPDATE CUSTOMER SET C_CURRENT_ADDR_SK = {placeholder} WHERE C_CUSTOMER_SK = {placeholder}"
+        )
+
+        # Item UPDATE - matches _update_item logic
+        statements.append(
+            f"-- TPC-DS Maintenance: Update {num_rows} item prices\n"
+            f"-- Same logic as MaintenanceOperations._update_item()\n"
+            f"UPDATE ITEM SET I_CURRENT_PRICE = {placeholder} WHERE I_ITEM_SK = {placeholder}"
+        )
+
+        # Inventory UPDATE - matches _update_inventory logic
+        statements.append(
+            f"-- TPC-DS Maintenance: Update {num_rows} inventory levels\n"
+            f"-- Same logic as MaintenanceOperations._update_inventory()\n"
+            f"UPDATE INVENTORY SET INV_QUANTITY_ON_HAND = {placeholder} "
+            f"WHERE INV_DATE_SK = {placeholder} AND INV_ITEM_SK = {placeholder} AND INV_WAREHOUSE_SK = {placeholder}"
+        )
+
+        return statements
+
+    def _generate_delete_sql_via_maintenance_ops(self, placeholder: str, num_rows: int) -> list[str]:
+        """Generate DELETE SQL using the SAME logic as actual execution.
+
+        These match the DELETE statements in MaintenanceOperations._delete_* methods.
+        """
+        statements = []
+
+        # The cutoff_date_sk matches _delete_old_sales
+        cutoff_date_sk = 2450815
+
+        # Delete old sales - matches _delete_old_sales logic
+        statements.append(
+            f"-- TPC-DS Maintenance: Delete {num_rows} old store sales records\n"
+            f"-- Same logic as MaintenanceOperations._delete_old_sales()\n"
+            f"DELETE FROM STORE_SALES WHERE SS_SOLD_DATE_SK < {cutoff_date_sk} LIMIT {num_rows // 3}"
+        )
+
+        statements.append(
+            f"-- TPC-DS Maintenance: Delete {num_rows} old catalog sales records\n"
+            f"DELETE FROM CATALOG_SALES WHERE CS_SOLD_DATE_SK < {cutoff_date_sk} LIMIT {num_rows // 3}"
+        )
+
+        statements.append(
+            f"-- TPC-DS Maintenance: Delete {num_rows} old web sales records\n"
+            f"DELETE FROM WEB_SALES WHERE WS_SOLD_DATE_SK < {cutoff_date_sk} LIMIT {num_rows // 3}"
+        )
+
+        # Delete old returns - matches _delete_old_returns logic
+        statements.append(
+            f"-- TPC-DS Maintenance: Delete {num_rows // 2} old store returns\n"
+            f"-- Same logic as MaintenanceOperations._delete_old_returns()\n"
+            f"DELETE FROM STORE_RETURNS WHERE SR_RETURNED_DATE_SK < {cutoff_date_sk} LIMIT {num_rows // 3}"
+        )
+
+        return statements
+
 
 # Aliases for backward compatibility
 MaintenanceTest = TPCDSMaintenanceTest

benchbox/core/tpcds/power_test.py

@@ -365,12 +365,20 @@ class TPCDSPowerTest:
 
             connection.close()
 
-            # Calculate Power@Size metric
+            # Calculate Power@Size metric (geometric mean per TPC spec)
             total_execution_time = time.time() - start_time
-
-            # Power@Size = 3600 * SF / Power_Test_Time (in seconds)
-            if total_execution_time > 0 and result.queries_successful > 0:
-                result.power_at_size = (3600.0 * self.config.scale_factor) / total_execution_time
+            exec_times = [
+                qr["execution_time"]
+                for qr in result.query_results
+                if qr.get("success", True) and qr.get("execution_time", 0) > 0
+            ]
+            if exec_times:
+                from benchbox.core.results.metrics import TPCMetricsCalculator
+
+                result.power_at_size = TPCMetricsCalculator.calculate_power_at_size(
+                    exec_times,
+                    self.config.scale_factor,
+                )
 
             result.total_time = total_execution_time
             result.end_time = datetime.now().isoformat()
@@ -496,18 +504,20 @@ class TPCDSPowerTest:
 
         return not result.power_at_size <= 0
 
-    def _calculate_power_at_size(self, total_time: float) -> float:
-        """Calculate Power@Size metric.
+    def _calculate_power_at_size(self, exec_times: list[float]) -> float:
+        """Calculate Power@Size metric using geometric mean.
 
         Args:
-            total_time: Total execution time in seconds
+            exec_times: Per-query execution times in seconds
 
         Returns:
-            Power@Size metric (3600 * SF / total_time)
+            Power@Size metric (3600 * SF / geometric_mean(exec_times))
         """
-        if total_time <= 0:
+        if not exec_times:
             return 0.0
-        return (3600.0 * self.scale_factor) / total_time
+        from benchbox.core.results.metrics import TPCMetricsCalculator
+
+        return TPCMetricsCalculator.calculate_power_at_size(exec_times, self.scale_factor)
 
     @property
     def query_sequence(self) -> list:

benchbox/core/tpcds/throughput_test.py

@@ -395,12 +395,14 @@ class TPCDSThroughputTest:
             streams = stream_manager.generate_streams()
             stream_queries = streams.get(stream_id, [])
 
-            # Execute queries for throughput testing (only main queries, not variants)
-            main_queries = [sq for sq in stream_queries if sq.variant is None]
+            # Execute all queries including multi-part variants (14a/b, 23a/b, 24a/b, 39a/b)
+            # TPC-DS specifies 99 query templates, but 4 of them (14, 23, 24, 39) have 2 parts each
+            # Total: 95 single queries + 8 variant queries = 103 queries per stream
+            all_queries = stream_queries
 
             # Apply query subset limit if configured
             if config.queries_per_stream is not None:
-                query_subset = main_queries[: min(config.queries_per_stream, len(main_queries))]
+                query_subset = all_queries[: min(config.queries_per_stream, len(all_queries))]
                 if config.verbose:
                     self.logger.info(
                         f"Stream {stream_id} using TPC-DS permutation with {len(query_subset)} queries "
@@ -408,7 +410,7 @@ class TPCDSThroughputTest:
                     )
             else:
                 # Execute all queries (full TPC-DS compliance)
-                query_subset = main_queries
+                query_subset = all_queries
                 if config.verbose:
                     self.logger.info(
                         f"Stream {stream_id} using TPC-DS permutation with {len(query_subset)} queries (full query set)"
@@ -416,9 +418,12 @@ class TPCDSThroughputTest:
 
             for position, stream_query in enumerate(query_subset):
                 query_id = stream_query.query_id
+                variant = stream_query.variant
+                # Display ID includes variant suffix (e.g., "14a", "23b")
+                query_display_id = f"{query_id}{variant}" if variant else str(query_id)
                 query_start = time.time()
                 query_result = {
-                    "query_id": query_id,
+                    "query_id": query_display_id,
                     "position": position + 1,
                     "stream_id": stream_id,
                     "execution_time": 0.0,
@@ -431,13 +436,26 @@ class TPCDSThroughputTest:
                     # Get the query with stream-specific parameters
                     # Use stream and position-specific seed as per TPC-DS specification
                     stream_seed = seed + stream_id * 1000 + position
-                    query_text = self.benchmark.get_query(
-                        query_id, seed=stream_seed, scale_factor=config.scale_factor, dialect=self.target_dialect
-                    )
+                    # Pass variant parameter for multi-part queries (14a/b, 23a/b, 24a/b, 39a/b)
+                    if variant is not None:
+                        query_text = self.benchmark.get_query(
+                            query_id,
+                            seed=stream_seed,
+                            scale_factor=config.scale_factor,
+                            variant=variant,
+                            dialect=self.target_dialect,
+                        )
+                    else:
+                        query_text = self.benchmark.get_query(
+                            query_id, seed=stream_seed, scale_factor=config.scale_factor, dialect=self.target_dialect
+                        )
 
                     # Execute the actual query to enable dry-run capture via platform adapter
-                    label = f"Stream_{stream_id}_Position_{position + 1}_Query_{query_id}"
+                    label = f"Stream_{stream_id}_Position_{position + 1}_Query_{query_display_id}"
                     try:
+                        # Set query context before execution for validation and error reporting
+                        if hasattr(connection, "set_query_context"):
+                            connection.set_query_context(query_display_id, stream_id=stream_id)
                         cursor = connection.execute(query_text)
                         # Fetch to complete execution in non-dry-run
                         if hasattr(cursor, "fetchall"):

benchbox/core/tpcds_obt/etl/transformer.py

@@ -5,6 +5,8 @@ from __future__ import annotations
 import json
 import logging
 import re
+import shutil
+import tempfile
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
@@ -79,7 +81,12 @@ class TPCDSOBTTransformer:
         output_path = output_dir / f"{OBT_TABLE_NAME}.{output_format}"
         manifest_path = output_dir / f"{OBT_TABLE_NAME}_manifest.json"
 
-        conn = self._connect()
+        # Use a file-backed temp database so DuckDB can spill to disk effectively
+        # during the large multi-channel UNION+JOIN transformation.
+        tmp_dir = tempfile.mkdtemp(prefix="benchbox_obt_")
+        tmp_db_path = Path(tmp_dir) / "obt.duckdb"
+
+        conn = self._connect(tmp_db_path)
         try:
             self._load_source_tables(conn, tpcds_dir, channel_list)
             union_sql = self._build_union_query(columns, channel_list)
@@ -107,14 +114,26 @@ class TPCDSOBTTransformer:
             return {"table": output_path, "manifest": manifest_path}
         finally:
             conn.close()
+            shutil.rmtree(tmp_dir, ignore_errors=True)
+
+    def _connect(self, db_path: Path | None = None) -> Any:
+        """Create a DuckDB connection configured for large ETL workloads.
 
-    def _connect(self) -> Any:
-        """Create a DuckDB connection."""
+        Args:
+            db_path: Path to a file-backed database. If None, uses in-memory.
+        """
+        target = str(db_path) if db_path else ":memory:"
         if self._duckdb is None:
             import duckdb
 
-            return duckdb.connect(":memory:")
-        return self._duckdb.connect(":memory:")
+            conn = duckdb.connect(target)
+        else:
+            conn = self._duckdb.connect(target)
+
+        # Configure for large-scale ETL: disable insertion-order preservation
+        # to reduce memory pressure during multi-way joins.
+        conn.execute("SET preserve_insertion_order = false")
+        return conn
 
     def _validate_channels(self, channels: Sequence[str]) -> None:
         """Validate requested channels."""

benchbox/core/tpch/dataframe_queries.py

@@ -839,6 +839,9 @@ def q21_expression_impl(ctx: DataFrameContext) -> Any:
     """TPC-H Q21: Suppliers Who Kept Orders Waiting (Expression Family).
 
     Identifies suppliers who delayed orders they could have filled.
+
+    Optimized using filter pushdown to compute expensive aggregates
+    only for relevant orders, avoiding the costly unique() + join pattern.
     """
     supplier = ctx.get_table("supplier")
     lineitem = ctx.get_table("lineitem")
@@ -849,59 +852,59 @@ def q21_expression_impl(ctx: DataFrameContext) -> Any:
 
     nation_name = "SAUDI ARABIA"
 
-    # Find late lineitems
-    l1 = lineitem.filter(col("l_receiptdate") > col("l_commitdate")).select(
-        col("l_orderkey").alias("l1_orderkey"),
-        col("l_suppkey").alias("l1_suppkey"),
+    # Step 1: Target suppliers from the specified nation (small set ~400)
+    target_suppliers = (
+        supplier.join(nation, left_on="s_nationkey", right_on="n_nationkey")
+        .filter(col("n_name") == lit(nation_name))
+        .select(col("s_suppkey").alias("target_suppkey"), col("s_name"))
     )
 
-    # Find orders with multiple suppliers
-    multi_supplier_orders = (
-        lineitem.group_by("l_orderkey")
-        .agg(col("l_suppkey").n_unique().alias("num_suppliers"))
-        .filter(col("num_suppliers") > lit(1))
-        .select("l_orderkey")
+    # Step 2: Valid orders with status 'F'
+    valid_orders = orders.filter(col("o_orderstatus") == lit("F")).select(col("o_orderkey").alias("valid_orderkey"))
+
+    # Step 3: Candidates - late lineitems from target suppliers on valid orders
+    # This is our candidate set - much smaller than full lineitem table
+    candidates = (
+        lineitem.filter(col("l_receiptdate") > col("l_commitdate"))
+        .join(target_suppliers, left_on="l_suppkey", right_on="target_suppkey")
+        .join(valid_orders, left_on="l_orderkey", right_on="valid_orderkey")
+        .select("l_orderkey", "l_suppkey", "s_name")
     )
 
-    # Find orders where other suppliers were late (for NOT EXISTS check)
-    other_late = lineitem.filter(col("l_receiptdate") > col("l_commitdate")).select(
-        col("l_orderkey").alias("l3_orderkey"),
-        col("l_suppkey").alias("l3_suppkey"),
+    # Step 4: Get candidate order keys for semi-join filtering
+    candidate_orders = candidates.select(col("l_orderkey").alias("cand_orderkey")).unique()
+
+    # Count distinct suppliers per order (only for candidate orders)
+    # This implements the EXISTS check: order has multiple suppliers
+    suppliers_per_order = (
+        lineitem.join(candidate_orders, left_on="l_orderkey", right_on="cand_orderkey", how="semi")
+        .group_by("l_orderkey")
+        .agg(col("l_suppkey").n_unique().alias("num_suppliers"))
+        .select(col("l_orderkey").alias("supp_orderkey"), col("num_suppliers"))
     )
 
-    # Main query with EXISTS/NOT EXISTS logic
-    # First build the base query up to the multi_supplier join
-    base = (
-        supplier.join(nation, left_on="s_nationkey", right_on="n_nationkey")
-        .filter(col("n_name") == lit(nation_name))
-        .join(l1, left_on="s_suppkey", right_on="l1_suppkey")
-        .join(orders, left_on="l1_orderkey", right_on="o_orderkey")
-        .filter(col("o_orderstatus") == lit("F"))
-        .join(multi_supplier_orders, left_on="l1_orderkey", right_on="l_orderkey", how="semi")
-    )
-
-    # For NOT EXISTS with correlated subquery, we need to:
-    # 1. Join with other_late
-    # 2. Filter where suppliers are different
-    # 3. Use anti join to exclude those orders
-    # Note: After join on s_suppkey=l1_suppkey, only s_suppkey remains
-    orders_with_other_late_suppliers = (
-        base.select("l1_orderkey", "s_suppkey")
-        .unique()
-        .join(other_late, left_on="l1_orderkey", right_on="l3_orderkey")
-        .filter(col("l3_suppkey") != col("s_suppkey"))
-        .select("l1_orderkey", "s_suppkey")
-        .unique()
+    # Count distinct LATE suppliers per order (only for candidate orders)
+    # This implements the NOT EXISTS check: no other supplier was late
+    late_suppliers_per_order = (
+        lineitem.filter(col("l_receiptdate") > col("l_commitdate"))
+        .join(candidate_orders, left_on="l_orderkey", right_on="cand_orderkey", how="semi")
+        .group_by("l_orderkey")
+        .agg(col("l_suppkey").n_unique().alias("num_late_suppliers"))
+        .select(col("l_orderkey").alias("late_orderkey"), col("num_late_suppliers"))
     )
 
+    # Step 5: Join candidates with counts and apply EXISTS/NOT EXISTS as filters
     result = (
-        base.join(
-            orders_with_other_late_suppliers,
-            on=["l1_orderkey", "s_suppkey"],
-            how="anti",
-        )
+        candidates.join(suppliers_per_order, left_on="l_orderkey", right_on="supp_orderkey")
+        .join(late_suppliers_per_order, left_on="l_orderkey", right_on="late_orderkey")
+        # EXISTS: order has multiple suppliers
+        .filter(col("num_suppliers") > lit(1))
+        # NOT EXISTS: only this supplier was late (no other late suppliers)
+        # If num_late_suppliers == 1, and we're a late supplier, no one else was late
+        .filter(col("num_late_suppliers") == lit(1))
+        # Final aggregation
         .group_by("s_name")
-        .agg(col("l1_orderkey").count().alias("numwait"))
+        .agg(col("l_orderkey").count().alias("numwait"))
         .sort(["numwait", "s_name"], descending=[True, False])
         .limit(100)
     )

benchbox/core/tpch/generator.py

@@ -26,6 +26,7 @@ from benchbox.utils.cloud_storage import CloudStorageGeneratorMixin, create_path
 from benchbox.utils.compression_mixin import CompressionMixin
 from benchbox.utils.data_validation import BenchmarkDataValidator
 from benchbox.utils.datagen_manifest import DataGenerationManifest, resolve_compression_metadata
+from benchbox.utils.file_format import COMPRESSION_EXTENSIONS
 from benchbox.utils.scale_factor import format_scale_factor
 from benchbox.utils.tpc_compilation import CompilationStatus, ensure_tpc_binaries
 from benchbox.utils.verbosity import VerbosityMixin, compute_verbosity
@@ -121,6 +122,10 @@ class TPCHDataGenerator(CompressionMixin, CloudStorageGeneratorMixin, VerbosityM
             try:
                 self._dbgen_exe = self._find_or_build_dbgen()
                 self.dbgen_available = self._dbgen_exe.exists()
+                # Update dbgen_path to the directory containing the executable
+                # This ensures dists.dss can be found when using precompiled binaries
+                if self.dbgen_available:
+                    self.dbgen_path = self._dbgen_exe.parent
             except (FileNotFoundError, RuntimeError, PermissionError) as exc:
                 self.dbgen_available = False
                 self._dbgen_error = exc
@@ -202,13 +207,19 @@ class TPCHDataGenerator(CompressionMixin, CloudStorageGeneratorMixin, VerbosityM
 
     def _raise_missing_dbgen(self) -> NoReturn:
         """Raise an error when dbgen is not available for data generation."""
+        import benchbox
+
+        pkg_dir = Path(benchbox.__file__).parent
+        binaries_path = pkg_dir / "_binaries/tpc-h"
+
         message = (
-            "TPC-H native tools are not bundled with this build. "
-            "Install the TPC-H toolkit and place the compiled binaries under "
-            f"{self._package_root / '_sources/tpc-h/dbgen'} or supply sample data."
+            "TPC-H dbgen tool not found. Pre-compiled binaries should be at "
+            f"{binaries_path}/<platform>/ but were not found. "
+            "This may indicate a corrupted or incomplete installation. "
+            "Try reinstalling: pip install --force-reinstall benchbox"
         )
         if self._dbgen_error:
-            message += f" Details: {self._dbgen_error}"
+            message += f"\n\nDetails: {self._dbgen_error}"
         raise RuntimeError(message)
 
     def _find_or_build_dbgen(self) -> Path:
@@ -766,9 +777,10 @@ class TPCHDataGenerator(CompressionMixin, CloudStorageGeneratorMixin, VerbosityM
 
     def _generate_local(self, output_dir: Path | None = None) -> dict[str, Path | list[Path]]:
         """Generate data locally (original implementation)."""
-        # Check if dbgen is available for data generation
-        if not self.dbgen_available:
-            self._raise_missing_dbgen()
+        # Trigger lazy loading of dbgen_exe, which will find precompiled binaries
+        # if sources are not available. This raises _raise_missing_dbgen() if neither
+        # sources nor precompiled binaries are found.
+        _ = self.dbgen_exe
 
         # Use provided output directory or fall back to instance output_dir
         target_dir = output_dir if output_dir is not None else self.output_dir
@@ -851,7 +863,8 @@ class TPCHDataGenerator(CompressionMixin, CloudStorageGeneratorMixin, VerbosityM
         for table, filename in table_files.items():
             # Check for compressed files first (regardless of current compression setting)
             # Data may have been generated with compression even if not requested now
-            for ext in [".zst", ".gz", ".lz4"]:
+            # Use centralized compression extensions for comprehensive coverage
+            for ext in COMPRESSION_EXTENSIONS:
                 compressed_file = target_dir / f"{filename}{ext}"
                 if compressed_file.exists():
                     existing[table] = compressed_file