sqlbenchdag 0.1.0__py3-none-any.whl

sql_benchmarks/scripts/sql/sort_spill/postgres/sort_wide_rows.sql

@@ -0,0 +1,26 @@
+-- Sort Spill: Multi-column sort on wide rows.
+-- Hypothesis: At some row count, the sort set exceeds work_mem and Postgres
+-- spills to disk, producing a visible performance cliff.
+-- The work_mem is set intentionally tight in pg_settings to expose this cliff.
+-- DuckDB (sort_spill/duckdb/) runs the same query as a control without this constraint.
+
+SELECT
+    id,
+    region,
+    category,
+    department,
+    sub_category,
+    ROUND(CAST(price AS NUMERIC), 4)        AS price,
+    ROUND(CAST(cost AS NUMERIC), 4)         AS cost,
+    ROUND(CAST(margin AS NUMERIC), 4)       AS margin,
+    quantity,
+    discount,
+    ROUND(CAST(price * quantity * (1.0 - discount) AS NUMERIC), 4) AS net_revenue
+FROM {{ sort_data_table }}
+ORDER BY
+    region ASC,
+    category ASC,
+    department ASC,
+    net_revenue DESC,
+    quantity DESC
+LIMIT 1000;

sql_benchmarks/scripts/sql/tpch/duckdb/q3_shipping_priority.sql

@@ -0,0 +1,25 @@
+-- TPC-H Query 3 (Shipping Priority)
+-- Modified for Jinja Templating
+SELECT
+    l.l_orderkey,
+    SUM(l.l_extendedprice * (1 - l.l_discount)) AS revenue,
+    o.o_orderdate,
+    o.o_shippriority
+FROM
+    {{ customer_table }} c,
+    {{ orders_table }} o,
+    {{ lineitem_table }} l
+WHERE
+    c.c_mktsegment = 'BUILDING'
+    AND c.c_custkey = o.o_custkey
+    AND l.l_orderkey = o.o_orderkey
+    AND o.o_orderdate < DATE '1995-03-15'
+    AND l.l_shipdate > DATE '1995-03-15'
+GROUP BY
+    l.l_orderkey,
+    o.o_orderdate,
+    o.o_shippriority
+ORDER BY
+    revenue DESC,
+    o.o_orderdate
+LIMIT 10;

sql_benchmarks/scripts/sql/tpch/postgres/q3_shipping_priority.sql

@@ -0,0 +1,25 @@
+-- TPC-H Query 3 (Shipping Priority)
+-- Modified for Jinja Templating
+SELECT
+    l.l_orderkey,
+    SUM(l.l_extendedprice * (1 - l.l_discount)) AS revenue,
+    o.o_orderdate,
+    o.o_shippriority
+FROM
+    {{ customer_table }} c,
+    {{ orders_table }} o,
+    {{ lineitem_table }} l
+WHERE
+    c.c_mktsegment = 'BUILDING'
+    AND c.c_custkey = o.o_custkey
+    AND l.l_orderkey = o.o_orderkey
+    AND o.o_orderdate < DATE '1995-03-15'
+    AND l.l_shipdate > DATE '1995-03-15'
+GROUP BY
+    l.l_orderkey,
+    o.o_orderdate,
+    o.o_shippriority
+ORDER BY
+    revenue DESC,
+    o.o_orderdate
+LIMIT 10;

sql_benchmarks/utils/common.py

@@ -0,0 +1,260 @@
+import os
+import glob
+import shutil
+import yaml
+import jinja2
+import jinja2.meta
+import numpy as np
+from ..constants import ACTIVE_CONFIG_PATH, ENGINE_SQL_DIALECTS, SQL_DIR
+from ..config_loader import ConfigLoader
+from typing import Dict, Any
+
+# Initialize the compiler once globally
+# NOTE: If this fails to initialize due to a strict violation, the error will propagate up 
+# when Dagster tries to load definitions, which is the correct fail-hard behavior.
+# ==========================================
+# 1. CONTEXT & CONFIG LOADING
+# ==========================================
+try:
+    _GLOBAL_COMPILER = ConfigLoader()
+except ValueError as e:
+    # Propagate structural errors during load time
+    raise e 
+
+def load_context() -> Dict[str, Any]:
+    """
+    Returns a consolidated context dictionary containing all derived and raw configuration
+    needed by asset factories. This replaces the old context loading logic.
+    """
+    raw_config = _GLOBAL_COMPILER.get_full_config()
+    
+    # --- Tracing All Necessary Context for Asset Factories ---
+    
+    # 1. Core Config Blocks
+    context = {
+        "full_config": raw_config,
+        "execution": _GLOBAL_COMPILER.execution,
+        "definitions": _GLOBAL_COMPILER.definitions,
+        "dataset_config": _GLOBAL_COMPILER.dataset, # Used by benchmark_factory for schema inference
+    }
+
+    # 2. Derived/Extracted Context (Crucial for Downstream Logic)
+    
+   # A. Active Engines (Used by benchmark_factory to loop over engines)
+    context["engines"] = context["execution"].get("engines", [])
+    
+    # B. Valid Tables (Set of table names for dependency checking)
+    table_defs = context["dataset_config"].get("tables", {})
+    context["tables"] = set(table_defs.keys())
+    
+    # C. Legacy Contract Fulfillment
+    context["table_defs"] = table_defs
+    
+    # D. Experiment Metadata
+    context["meta"] = raw_config.get("meta", {})
+    
+    # E. Full Scenario Config
+    context["scenario_config"] = _GLOBAL_COMPILER.scenario_config
+    
+    return context
+
+def get_target_sql_dir(config):
+    suite = config.get("execution", {}).get("test_suite", "")
+    return os.path.join(SQL_DIR, suite) if suite else SQL_DIR
+
+# ==========================================
+# 2. SCHEMA PARSING
+# ==========================================
+def get_tables_used_in_sql(sql_path, valid_tables_set):
+    """Parses SQL template to find dependencies ({{ table_name }})."""
+    with open(sql_path, "r") as f:
+        raw_template = f.read()
+
+    env = jinja2.Environment()
+    try:
+        ast = env.parse(raw_template)
+        required_vars = jinja2.meta.find_undeclared_variables(ast)
+    except Exception as e:
+        print(f"Jinja Parse Error {sql_path}: {e}")
+        return [], raw_template
+
+    used_tables = [
+        var.replace("_table", "") 
+        for var in required_vars 
+        if var.endswith("_table") and var.replace("_table", "") in valid_tables_set
+    ]
+    return used_tables, raw_template
+
+def extract_foreign_keys(table_def):
+    """Returns list of dicts: [{'col': 'x', 'target': 'y', 'target_col': 'z'}]"""
+    fks = []
+    for col in table_def.get('columns', []):
+        if col.get('provider') == 'foreign_key':
+            fks.append({
+                'col': col['name'],
+                'target': col.get('target_table'),
+                'target_col': col.get('target_column')
+            })
+    return fks
+
+def get_data_dependencies(table_name, table_configs):
+    """Returns upstream dependencies for a specific table."""
+    deps = set()
+    t_conf = table_configs.get(table_name, {})
+    
+    # Implicit FK deps
+    for fk in extract_foreign_keys(t_conf):
+        if fk['target']: deps.add(fk['target'])
+                
+    # Explicit deps
+    for d in t_conf.get('deps', []): deps.add(d)
+        
+    return list(deps)
+
+# ==========================================
+# 3. MATH & NORMALIZATION
+# ==========================================
+def normalize_distribution(options: list, weights: list):
+    """
+    Robustly normalizes weights for numpy generation.
+    Used by BOTH Data Generator (Plugin) and Metadata (Dashboard).
+    """
+    if len(options) != len(weights):
+        raise ValueError(f"Mismatch: {len(options)} options vs {len(weights)} weights")
+        
+    w_arr = np.array(weights, dtype=float)
+    if w_arr.sum() <= 0: raise ValueError("Sum of weights must be positive")
+    
+    # Normalize to sum to 1.0
+    norm_weights = w_arr / w_arr.sum()
+    
+    # THE FIX: Handle floating point drift (numpy strict division)
+    # Guaranteed to sum to exactly 1.0
+    if len(norm_weights) > 0:
+        norm_weights[-1] = 1.0 - norm_weights[:-1].sum()
+        
+    return options, norm_weights
+
+# ==========================================
+# 4. METADATA INFERENCE
+# ==========================================
+def infer_metadata_from_sql(sql_content, dataset_config):
+    """Scans SQL for known data keys to derive metadata."""
+    mapping = {}
+    tables = dataset_config.get('tables', {})
+    
+    # Build Value Map
+    if isinstance(tables, dict):
+        for table_def in tables.values():
+            for col in table_def.get('columns', []):
+                if 'options' in col and 'weights' in col:
+                    try:
+                        opts, probs = normalize_distribution(col['options'], col['weights'])
+                        for opt, p in zip(opts, probs):
+                            mapping[str(opt)] = float(p)
+                    except ValueError: continue 
+
+    # Scan SQL
+    meta = {}
+    for key, weight in mapping.items():
+        if f"'{key}'" in sql_content or f'"{key}"' in sql_content:
+            meta['selectivity_pct'] = weight * 100.0
+            meta['data_slice'] = key
+            break 
+    return meta
+
+# ==========================================
+# 5. PARTITION KEYS
+# ==========================================
+
+def generate_partition_keys(matrix_config):
+    """
+    Takes a matrix dict and returns sorted partition keys.
+    Used by: sensors.py, run_experiment.py (CLI), and potentially partitions.py
+    """
+    import itertools
+    
+    if not matrix_config:
+        return []
+
+    # 1. Sort keys to ensure consistent naming
+    keys = sorted(matrix_config.keys())
+    values = [matrix_config[k] for k in keys]
+    
+    # 2. Cartesian Product
+    partition_keys = []
+    for combination in itertools.product(*values):
+        key_str = "_".join(str(v) for v in combination)
+        partition_keys.append(key_str)
+        
+    return partition_keys
+
+# ==========================================
+# 6. ASSET NAMING UTILITY 
+# ==========================================
+
+def get_engine_asset_prefix(engine_name: str) -> str:
+    """
+    Resolves the engine resource key ('postgres', 'duckdb')
+    to the canonical asset prefix ('pg_', 'duckdb_').
+    This is the single source of truth for asset naming conventions across all factories.
+    """
+    if engine_name == 'postgres':
+        return 'pg_'
+    # Default: Use the engine name itself followed by an underscore
+    return f'{engine_name}_'
+
+def get_engine_sql_dialect(engine_name: str) -> str:
+    """
+    Resolves an engine resource key to the SQL scenario directory it executes
+    (sql/<suite>/<dialect>/). The single source of truth for dialect reuse.
+    """
+    return ENGINE_SQL_DIALECTS.get(engine_name, engine_name)
+
+def copy_suite_queries(capsule_dir: str) -> int:
+    """Embed into the capsule (as queries/) the exact SQL that ran: for each
+    engine in the capsule's config, the .sql files of its dialect directory.
+    This is the subset of the suite the experiment's engines actually used — a
+    reader's convenience copy, NOT a hash input. The Experiment ID hashes the
+    full suite from source, independently of the capsule. Returns file count."""
+    cfg_path = os.path.join(capsule_dir, "experiment_config.yaml")
+    if not os.path.exists(cfg_path):
+        return 0
+    with open(cfg_path) as f:
+        config = yaml.safe_load(f) or {}
+    execution = config.get("execution", {})
+    if not execution.get("test_suite"):
+        return 0
+    dialects = {get_engine_sql_dialect(e) for e in execution.get("engines", [])}
+    src = get_target_sql_dir(config)
+    dest = os.path.join(capsule_dir, "queries")
+    if os.path.isdir(dest):
+        shutil.rmtree(dest)
+    written = 0
+    for dialect in sorted(dialects):
+        dialect_dir = os.path.join(src, dialect)
+        if not os.path.isdir(dialect_dir):
+            continue
+        for sql_file in sorted(glob.glob(os.path.join(dialect_dir, "*.sql"))):
+            out = os.path.join(dest, dialect, os.path.basename(sql_file))
+            os.makedirs(os.path.dirname(out), exist_ok=True)
+            shutil.copy2(sql_file, out)
+            written += 1
+    return written
+
+def get_engine_benchmark_group(engine_name: str) -> str:
+    """
+    Dagster group name for an engine's benchmark assets. Shared by the
+    benchmark factory (asset creation) and execute_run (asset selection) so
+    the two can never drift apart.
+    """
+    return f"dynamic_bench_{engine_name}"
+
+def get_scoped_asset_name(base_name: str, exp_id: str) -> str:
+    """
+    Generates a globally unique asset name prefixed by the experiment ID.
+    Format: e_<exp_id>__<base_name>
+    """
+    if not exp_id or exp_id == "unknown":
+        return base_name
+    return f"e_{exp_id}__{base_name}"

sql_benchmarks/utils/ddl.py

@@ -0,0 +1,47 @@
+# We need to import the parser helper from common to avoid code duplication
+# But to avoid circular imports, we just reimplement the tiny extractor or 
+# pass the FKs in. Let's keep it simple and self-contained.
+
+class PostgresDDLGenerator:
+    def __init__(self, table_def, physical_name, partition_key):
+        self.def_ = table_def
+        self.name = physical_name
+        self.part = partition_key
+
+    def generate_pk_sql(self):
+        pk_cols = [c['name'] for c in self.def_.get('columns', []) if c.get('primary_key')]
+        if pk_cols:
+            return f"ALTER TABLE {self.name} ADD PRIMARY KEY ({', '.join(pk_cols)});"
+        return None
+
+    def generate_index_sqls(self):
+        sqls = []
+        for idx in self.def_.get('indexes', []):
+            cols = idx.get('columns', [])
+            if not cols: continue
+            # Index names are schema-global in Postgres, but the same table_def is
+            # applied to every partition's physical table. A fixed config name would
+            # collide across partitions — and CREATE INDEX IF NOT EXISTS would
+            # SILENTLY skip all but the first. Scope the name to the physical table
+            # so it is unique, and omit IF NOT EXISTS so a genuine collision is loud.
+            base = idx.get('name') or f"idx_{'_'.join(cols)}"
+            name = f"{base}_{self.name}"
+            sqls.append(f"CREATE INDEX {name} ON {self.name} ({', '.join(cols)});")
+        return sqls
+
+    def generate_fk_sqls(self):
+        sqls = []
+        # Local logic to find FKs (Clean and self-contained)
+        for col in self.def_.get('columns', []):
+            if col.get('provider') == 'foreign_key':
+                target = col.get('target_table')
+                target_col = col.get('target_column')
+                if not target or not target_col: continue
+
+                target_phys = f"{target}_{self.part}"
+                fk_name = f"fk_{self.name}_{col['name']}"
+                sqls.append(
+                    f"ALTER TABLE {self.name} ADD CONSTRAINT {fk_name} "
+                    f"FOREIGN KEY ({col['name']}) REFERENCES {target_phys} ({target_col});"
+                )
+        return sqls

sql_benchmarks/utils/hasher.py

@@ -0,0 +1,124 @@
+import hashlib
+import json
+import os
+import re
+import ast
+from .common import get_target_sql_dir
+
+def normalize_sql(content):
+    """
+    Standardizes SQL for hashing using Regex.
+    Robust to Agent formatting, comments, and Jinja templates.
+    """
+    # 1. Remove Block Comments /* ... */
+    content = re.sub(r'/\*[\s\S]*?\*/', ' ', content)
+    
+    # 2. Remove Line Comments -- ...
+    content = re.sub(r'--.*', ' ', content)
+    
+    # 3. Collapse Whitespace (Handling newlines/tabs)
+    content = " ".join(content.split())
+    
+    # 4. Remove Trailing Semicolon (
+    content = content.strip().rstrip(';').strip()
+    
+    # 5. Canonicalize Case (Select -> select)
+    return content.lower()
+
+def normalize_python(content):
+    """
+    Parses Python code to AST and regenerates it.
+    Ignores comments, docstrings, and formatting.
+    """
+    try:
+        tree = ast.parse(content)
+        for node in ast.walk(tree):
+            if not isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef, ast.Module)):
+                continue
+            if not (node.body and isinstance(node.body[0], ast.Expr)):
+                continue
+            val = node.body[0].value
+            if isinstance(val, ast.Constant) and isinstance(val.value, str):
+                node.body.pop(0) # Remove Docstring
+        return ast.unparse(tree)
+    except SyntaxError:
+        return content
+
+def generate_experiment_hash(config_dict, root_dir):
+    hasher = hashlib.sha256()
+
+    # 1. Hash Config
+    clean_config = {k: v for k, v in config_dict.items() if k != 'meta'}
+    config_str = json.dumps(clean_config, sort_keys=True)
+    hasher.update(config_str.encode('utf-8'))
+
+    def hash_folder(folder_path, extension, normalizer=None, exclude_dirs=()):
+        if not os.path.exists(folder_path): return
+
+        for root, dirs, files in os.walk(folder_path):
+            dirs[:] = sorted(d for d in dirs if d not in exclude_dirs)
+            for file in sorted(files):
+                if file.endswith(extension):
+                    rel_path = os.path.relpath(os.path.join(root, file), root_dir)
+                    hasher.update(rel_path.encode('utf-8'))
+
+                    with open(os.path.join(root, file), 'r', encoding='utf-8') as f:
+                        content = f.read()
+
+                    if normalizer:
+                        content = normalizer(content)
+
+                    hasher.update(content.encode('utf-8'))
+
+    # 2. Hash SQL Files
+    target_sql_dir = get_target_sql_dir(config_dict)
+    hash_folder(target_sql_dir, ".sql", normalizer=normalize_sql)
+
+    # 3. Hash ALL Python that can change what a measurement means: the whole
+    #    package — orchestration (assets/), engines (resources/), data
+    #    generators (plugins/), and the root/utils machinery (config_loader
+    #    assembles engine_params; system.py owns the cold-cache primitive).
+    #    Excluded: api/ only READS results — it cannot affect a measurement —
+    #    and experiments/ holds configs/results, which are hashed separately.
+    #    The ID fingerprints the QUESTION; runtime conditions (engine
+    #    versions, hardware) are recorded in the capsule metadata instead.
+    hash_folder(
+        os.path.join(root_dir, "sql_benchmarks"),
+        ".py",
+        normalizer=normalize_python,
+        exclude_dirs=("api", "experiments", "__pycache__"),
+    )
+
+    return hasher.hexdigest()[:8]
+
+def generate_integrity_seal(results_dir):
+    """
+    Generates a SHA-256 seal for the entire results capsule: every file in
+    the capsule folder (CSV, fragments, metadata, config, dashboard), the
+    seal file itself excluded. Detects silent modification of published
+    results; it is a checksum, not a signature — code provenance is the
+    Experiment ID's job, not this seal's.
+    """
+    hasher = hashlib.sha256()
+    
+    # Walk results_dir and hash all files deterministically
+    for root, dirs, files in os.walk(results_dir):
+        # Sort files to ensure deterministic hashing
+        for file in sorted(files):
+            # Skip the seal and its sidecars (timestamp proof, signature):
+            # they are computed FROM the seal, so they must not feed back into it.
+            if file == "integrity.seal" or file.startswith("integrity.seal."):
+                continue
+            
+            file_path = os.path.join(root, file)
+            rel_path = os.path.relpath(file_path, results_dir)
+            
+            # Hash metadata
+            hasher.update(rel_path.encode('utf-8'))
+            
+            # Hash content
+            with open(file_path, 'rb') as f:
+                while chunk := f.read(8192):
+                    hasher.update(chunk)
+                    
+    return hasher.hexdigest()

sql_benchmarks/utils/integrity_monitor.py

@@ -0,0 +1,55 @@
+import os
+import hashlib
+
+class IntegrityMonitor:
+    """
+    Monitors a directory for any changes after an initial snapshot.
+    Used to ensure the 'Clean Room' staging area remains isolated.
+    """
+    def __init__(self, target_dir):
+        self.target_dir = target_dir
+        self.initial_state = self._compute_state()
+
+    def _compute_state(self):
+        state = {}
+        for root, _, files in os.walk(self.target_dir):
+            for file in files:
+                path = os.path.join(root, file)
+                rel_path = os.path.relpath(path, self.target_dir)
+                
+                # Ignore runtime artifacts, transient directories, and experiment outputs
+                if any(x in rel_path for x in ["__pycache__", ".pyc", "experiments/", ".DS_Store"]):
+                    continue
+                    
+                try:
+                    with open(path, "rb") as f:
+                        state[rel_path] = hashlib.sha256(f.read()).hexdigest()
+                except Exception:
+                    continue
+        return state
+
+    def check_drift(self):
+        """Returns a list of files that have been modified, added, or deleted."""
+        current_state = self._compute_state()
+        drift = []
+        
+        # Check for modifications and deletions
+        for path, original_hash in self.initial_state.items():
+            if path not in current_state:
+                drift.append(f"DELETED: {path}")
+            elif current_state[path] != original_hash:
+                curr_hash = current_state[path]
+                drift.append(f"MODIFIED: {path} (Expected: {original_hash[:8]}... Got: {curr_hash[:8]}...)")
+        
+        # Check for additions
+        for path in current_state:
+            if path not in self.initial_state:
+                drift.append(f"ADDED: {path}")
+                
+        return drift
+
+    def get_current_hash(self) -> str:
+        """Returns a single aggregate hash of the current directory state."""
+        state = self._compute_state()
+        combined = "".join(sorted(state.values()))
+        return hashlib.sha256(combined.encode()).hexdigest()