sqlbenchdag 0.1.0__py3-none-any.whl

sql_benchmarks/config_loader.py

@@ -0,0 +1,104 @@
+import os
+import yaml
+import itertools
+from typing import Dict, Any, Tuple, List
+from .constants import ACTIVE_CONFIG_PATH
+
+class ConfigLoader:
+    def __init__(self, config_path: str = ACTIVE_CONFIG_PATH):
+        self.config_path = config_path
+        self._raw_config: Dict[str, Any] = {}
+        self.execution: Dict[str, Any] = {}
+        self.definitions: Dict[str, Any] = {}
+        self.dataset: Dict[str, Any] = {}
+        
+        # Public, consolidated artifacts
+        self.scenario_config: Dict[str, Dict[str, Any]] = {}
+        self.partition_keys: List[str] = []
+
+        self._load_and_validate()
+        self._compile_scenario_config()
+
+    def _load_and_validate(self) -> None:
+        """Loads YAML and performs initial structural checks."""
+        if not os.path.exists(self.config_path):
+            return
+
+        try:
+            with open(self.config_path, "r") as f:
+                self._raw_config = yaml.safe_load(f) or {}
+        except Exception as e:
+            raise ValueError(f"CRITICAL: Failed to parse {self.config_path}: {e}")
+
+        # --- STRICT SCHEMA & SEMANTIC VALIDATION ---
+        from .validator import ExperimentValidator
+        ExperimentValidator.validate(self._raw_config, source_label=self.config_path)
+
+        self.execution = self._raw_config.get("execution", {})
+        self.definitions = self._raw_config.get("definitions", {})
+        self.dataset = self._raw_config.get("dataset", {})
+
+    def _compile_scenario_config(self) -> None:
+        """
+        Translates the symbolic matrix into numeric parameters and generates partition keys.
+        This is the consolidated logic from partitions.py.
+        """
+        if "matrix" not in self.execution:
+             raise ValueError("CRITICAL: Experiment must define a 'matrix' strictly under 'execution.matrix'.")
+             
+        matrix = self.execution["matrix"]
+        keys = sorted(list(matrix.keys()))
+        symbolic_values = [matrix[k] for k in keys]
+
+        for symbolic_combination in itertools.product(*symbolic_values):
+            
+            # A. Create the Symbolic Partition Key (e.g., 'tiny_ssd')
+            key_str = "_".join(str(v) for v in symbolic_combination)
+            self.partition_keys.append(key_str)
+            
+            # B. Translate to Numeric/Literal Values for Execution (The Payload)
+            numeric_params = {}
+            for dim_name, symbolic_value in zip(keys, symbolic_combination):
+                definition_map = self.definitions.get(dim_name, {})
+                
+                # 1. Try to resolve the alias using the definition map
+                if symbolic_value in definition_map:
+                    numeric_value = definition_map[symbolic_value]
+                
+                # 2. STRICT VALIDATION (Non-hardcoded): Fail if expected alias is missing
+                elif isinstance(symbolic_value, str) and definition_map and symbolic_value not in definition_map:
+                     raise ValueError(
+                        f"STRICT VIOLATION: Alias '{symbolic_value}' in matrix dimension '{dim_name}' "
+                        f"could not be resolved. Definition block 'definitions.{dim_name}' exists "
+                        "but is missing this alias."
+                     )
+                
+                # 3. Otherwise, the value is a literal (e.g., 'ssd', 1000).
+                else:
+                    numeric_value = symbolic_value
+                
+                numeric_params[dim_name] = numeric_value
+
+            # C. Assemble namespaced engine params: the static execution.engine_params
+            # block merged with namespaced matrix dimensions ('postgres.work_mem'
+            # -> engine_params['postgres']['work_mem']). Varied dimensions override
+            # static values. Stored as a nested key so consumers never have to
+            # derive it themselves; the factory hands each engine ONLY its own
+            # namespace at run time.
+            engine_params = {
+                ns: dict(settings)
+                for ns, settings in (self.execution.get("engine_params") or {}).items()
+            }
+            for dim_name, value in numeric_params.items():
+                if "." in dim_name:
+                    ns, param = dim_name.split(".", 1)
+                    engine_params.setdefault(ns, {})[param] = value
+            if engine_params:
+                numeric_params["engine_params"] = engine_params
+
+            # D. Store the Numeric/Literal parameters under the Symbolic Key
+            self.scenario_config[key_str] = numeric_params
+    
+    def get_full_config(self) -> Dict[str, Any]:
+        """Returns the full parsed configuration."""
+        return self._raw_config

sql_benchmarks/constants.py

@@ -0,0 +1,60 @@
+import os
+
+# 1. ANCHOR
+CURRENT_FILE_PATH = os.path.abspath(__file__)
+
+# 2. ZONES
+PACKAGE_DIR = os.path.dirname(CURRENT_FILE_PATH)        # sql_benchmarks/
+ROOT_DIR = os.path.dirname(PACKAGE_DIR)                 # sql-benchmarks-dagster/
+
+# 3. SUB-DIRECTORIES (Environment-Aware Redirection)
+# Fallback to absolute paths relative to ROOT_DIR
+EXPERIMENTS_DIR = os.path.join(PACKAGE_DIR, "experiments")
+SCRIPTS_DIR = os.path.join(PACKAGE_DIR, "scripts")
+SQL_DIR = os.path.join(SCRIPTS_DIR, "sql")
+
+# OUTPUT REDIRECTION (Used for Zero-Copy Isolation)
+DATA_DIR = os.getenv("SB_DATA_DIR", os.path.join(ROOT_DIR, "data"))
+RESULTS_DIR = os.getenv("SB_RESULTS_DIR", os.path.join(EXPERIMENTS_DIR, "results"))
+VIOLATIONS_DIR = os.getenv("SB_VIOLATIONS_DIR", os.path.join(EXPERIMENTS_DIR, "violations"))
+REPORTS_DIR = os.getenv("SB_REPORTS_DIR", os.path.join(EXPERIMENTS_DIR, "reports"))
+
+# 4. FILES
+ACTIVE_CONFIG_PATH = os.getenv("ACTIVE_CONFIG_PATH", os.path.join(EXPERIMENTS_DIR, "active.yaml"))
+CONFIG_ARCHIVE_DIR = os.path.join(EXPERIMENTS_DIR, "configs")
+PROCESSED_SUFFIX = ".processed"
+EXPERIMENT_EXTENSIONS = (".yaml", ".yml")
+
+# 5. ENGINE REGISTRIES
+# Convention: cross-engine registries live HERE; engine-specific vocabulary
+# allowlists (PG_SETTING_KEYS, DUCKDB_SETTING_KEYS) stay colocated with their
+# engine clients in resources/.
+#
+# Engines whose SQL dialect is another engine's: they reuse that engine's
+# scenario directory (sql/<suite>/<dialect>/) instead of duplicating SQL.
+# Quack is DuckDB served over a client-server protocol — identical dialect,
+# different transport.
+ENGINE_SQL_DIALECTS = {
+    "quack": "duckdb",
+    "quack_pushdown": "duckdb",
+}
+
+# Every engine the harness can run. A drift test asserts this matches the
+# resources actually registered in definitions.py — add an engine there
+# without updating this list and CI fails loudly.
+KNOWN_ENGINES = ["actian", "duckdb", "postgres", "quack", "quack_pushdown", "typedb"]
+
+# The lab's maker's mark — carried in every capsule's `generator` field and on
+# every release tag (sqlbenchdag-<topic>-v<N>-<YYYYMMDD>). sql + bench + dag
+# (Dagster). Verified unique on GitHub/PyPI; chosen once, do not change.
+LAB_SLUG = "sqlbenchdag"
+
+# 6. DAGSTER CONFIG
+_package_name = os.path.basename(PACKAGE_DIR) 
+DAGSTER_MODULE_TARGET = f"{_package_name}.definitions"
+
+# 6. EXECUTION TUNING
+DEFAULT_CHUNK_SIZE = int(os.getenv("SB_CHUNK_SIZE", 500_000))
+
+# 7. SAFETY
+AUDIT_LOCK_PATH = os.path.join(ROOT_DIR, "audit.lock")

sql_benchmarks/coordinator.py

@@ -0,0 +1,282 @@
+import os
+import yaml
+import shutil
+import time
+import subprocess
+import sys
+import json
+from .validator import ExperimentValidator
+from .constants import ROOT_DIR, CONFIG_ARCHIVE_DIR, EXPERIMENTS_DIR, PROCESSED_SUFFIX, RESULTS_DIR, VIOLATIONS_DIR, REPORTS_DIR, AUDIT_LOCK_PATH, ACTIVE_CONFIG_PATH
+from .utils.hasher import generate_experiment_hash, generate_integrity_seal
+from .utils.common import copy_suite_queries
+from .utils.semantic_auditor import SemanticAuditor
+from .harness import IsolationHarness
+
+class ExperimentCoordinator:
+    """
+    Orchestrates the Zero-Copy experiment lifecycle:
+    Validation -> Redirection -> Execution -> Monitoring -> Commitment
+    """
+    
+    
+    def __init__(self, target_yaml: str, headless: bool = False):
+        self.target_yaml = target_yaml
+        self.headless = headless
+        self.config = None
+        self.exp_id = None
+        # Raw source bytes, captured at validation — the exact text that was
+        # parsed and hashed. Archived verbatim into the capsule so the stored
+        # config is the author's file, not a re-serialization of it.
+        self._source_yaml = None
+
+    def run(self) -> bool:
+        # 0. Safety Check
+        if os.path.exists(AUDIT_LOCK_PATH):
+            print("[CRITICAL] AUDIT LOCK ACTIVE. Experiment aborted for safety.")
+            return False
+
+        # Phase 1: STRICT VALIDATION
+        try:
+            with open(self.target_yaml, "r") as f:
+                self._source_yaml = f.read()
+            self.config = yaml.safe_load(self._source_yaml)
+
+            ExperimentValidator.validate(self.config, source_label=os.path.basename(self.target_yaml))
+            
+            # Derive Identity (STRICT SHA-BASED)
+            self.exp_id = generate_experiment_hash(self.config, ROOT_DIR)
+            
+            self.config["meta"] = self.config.get("meta", {})
+            self.config["meta"]["experiment_id"] = self.exp_id
+            
+            # Check Registry
+            if os.path.exists(os.path.join(CONFIG_ARCHIVE_DIR, f"config_{self.exp_id}.yaml")):
+                print(f"[INFO] SKIPPING: Experiment {self.exp_id} already exists in registry.")
+                return True
+                
+        except Exception as e:
+            print(f"[REJECTED] Experiment contract failed validation: {e}")
+            return False
+
+        # Phase 2: PREPARE EXECUTION (Isolated)
+        #
+        # Write the canonical active.yaml FIRST (with experiment_id injected).
+        # This is the single source of truth: every component that reads
+        # active.yaml will see the same config, and the file can be traced back
+        # to the exact experiment being run.
+        with open(ACTIVE_CONFIG_PATH, 'w') as f:
+            yaml.dump(self.config, f, sort_keys=False)
+        print(f"[INFO] active.yaml updated → experiment_id: {self.exp_id}")
+
+        harness = IsolationHarness(self.exp_id)
+        redirects = harness.provision()
+        os.environ.update(redirects)
+
+        # Point the scratchpad's active.yaml at the same config so that
+        # subprocesses running inside the scratchpad read the correct experiment.
+        active_path = os.path.join(redirects["SCRATCHPAD_ROOT"], "active.yaml")
+        os.environ["ACTIVE_CONFIG_PATH"] = active_path
+        os.makedirs(os.path.dirname(active_path), exist_ok=True)
+        with open(active_path, 'w') as f:
+            yaml.dump(self.config, f, sort_keys=False)
+
+        # Phase 3: EXECUTION
+        print(f"[INFO] Executing {self.exp_id} in isolated scratchpad...")
+        
+        try:
+            # Prepare environment
+            local_env = os.environ.copy()
+            success = self._execute_direct(local_env)
+            
+            if not success:
+                print(f"[FAILURE] Technical execution failed.")
+                return False
+
+            # Phase 4: CODE-DRIFT GATE
+            # The Experiment ID was computed from the code at submission; if
+            # the package changed during execution, the ID no longer names
+            # what actually ran. Refuse to finalize — loudly.
+            drift = harness.check_integrity()
+            if drift:
+                print(f"[CRITICAL] Code drift detected during execution — results NOT committed:")
+                for d in drift:
+                    print(f"           {d}")
+                return False
+
+            # Phase 5: FINAL VERIFICATION & REGISTRY
+            return self._finalize_results()
+            
+        finally:
+            harness.cleanup()
+
+    def _execute_direct(self, local_env: dict) -> bool:
+        from .utils.common import generate_partition_keys
+        
+        # Generate Partition Keys
+        matrix = self.config.get("execution", {}).get("matrix") or self.config.get("execution", {}).get("dimensions")
+        keys = generate_partition_keys(matrix)
+        
+        overall_success = True
+        keys = keys if keys else [None]
+            
+        for pk in keys:
+            cmd = [sys.executable, "execute_run.py"]
+            if pk:
+                print(f"       -> Partition: {pk}")
+                cmd.extend(["--partition", pk])
+            else:
+                cmd.append("--all")
+            
+            p = subprocess.run(cmd, cwd=ROOT_DIR, env=local_env)
+            if p.returncode != 0:
+                overall_success = False
+        
+        # Final Reporting
+        cmd_report = [sys.executable, "execute_run.py", "--reporting"]
+        p_report = subprocess.run(cmd_report, cwd=ROOT_DIR, env=local_env)
+        
+        return overall_success and p_report.returncode == 0
+
+    def _finalize_results(self) -> bool:
+        """
+        Verifies results in the scratchpad, then commits them to the canonical
+        results directory.
+
+        The scratchpad env var (SB_RESULTS_DIR) is set by the harness AFTER
+        coordinator constants are imported, so the module-level RESULTS_DIR
+        still points to the real experiments/results/ dir.  We therefore look
+        for results in the scratchpad first, then copy them to the canonical dir.
+        """
+        # The subprocess wrote results here (env-redirected scratchpad)
+        scratchpad_results = os.environ.get("SB_RESULTS_DIR", RESULTS_DIR)
+        scratch_exp_folder = os.path.join(scratchpad_results, self.exp_id)
+
+        # Canonical destination (the real experiments/results/ dir)
+        canonical_exp_folder = os.path.join(RESULTS_DIR, self.exp_id)
+
+        csv_target = os.path.join(scratch_exp_folder, f"{self.exp_id}.csv")
+        dashboard_target = os.path.join(scratch_exp_folder, f"{self.exp_id}.html")
+
+        if not os.path.exists(csv_target) and not os.path.exists(dashboard_target):
+            print(f"[ERROR] Run finished but no results found (Checked {csv_target} and {dashboard_target})")
+            return False
+
+        # 1. Capture Metadata (in scratchpad first)
+        from .utils.system import capture_environment, generator_id
+        metadata = {
+            "experiment_id": self.exp_id,
+            "timestamp": time.time(),
+            "config_id": f"config_{self.exp_id}",
+            # Conditions, not identity: the bench this question was answered on
+            "environment": capture_environment(),
+            # Effective data seed (declarative_gen): explicit even when the
+            # YAML omits it — a capsule must state the seed its data came
+            # from, never leave it implied by a code default.
+            "dataset_seed": (self.config.get("dataset") or {}).get("seed", 42),
+            # Maker's mark: which tool + build produced this capsule.
+            "generator": generator_id(),
+        }
+        with open(os.path.join(scratch_exp_folder, f"metadata_{self.exp_id}.json"), "w") as f:
+            json.dump(metadata, f, indent=4)
+
+        # 1.5 Semantic Audit
+        auditor = SemanticAuditor()
+        violations = []
+        fragments_dir = os.path.join(scratch_exp_folder, "fragments")
+
+        if os.path.exists(fragments_dir):
+            for filename in os.listdir(fragments_dir):
+                file_path = os.path.join(fragments_dir, filename)
+                if filename.endswith(".json"):
+                    with open(file_path, 'r') as f:
+                        try:
+                            data = json.load(f)
+                            audit_res = auditor.audit_fragment(data)
+                            if not audit_res["success"]:
+                                violations.append(f"JSON {filename} failed audit: {audit_res['violations']}")
+                        except json.JSONDecodeError as e:
+                            violations.append(f"JSON {filename} is malformed: {e}")
+
+        is_semantically_valid = len(violations) == 0
+        if not is_semantically_valid:
+            print(f"[WARNING] Semantic Violation Detected in {self.exp_id}: {violations}")
+            violation_dest = os.path.join(VIOLATIONS_DIR, self.exp_id)
+            os.makedirs(violation_dest, exist_ok=True)
+            shutil.copy(csv_target, os.path.join(violation_dest, "results.csv"))
+            return False
+
+        # 2. Commit scratchpad → canonical results dir
+        # Guard: source and destination must be disjoint trees. A destination
+        # nested inside its source turns copytree into infinite recursive
+        # nesting (historical incident: 5K+ file explosion -> OOM).
+        src_real = os.path.realpath(scratch_exp_folder)
+        dst_real = os.path.realpath(canonical_exp_folder)
+        if src_real.startswith(dst_real + os.sep) or dst_real.startswith(src_real + os.sep):
+            raise RuntimeError(
+                f"REFUSED: nested copy {src_real} <-> {dst_real} would recurse infinitely."
+            )
+        if scratch_exp_folder != canonical_exp_folder:
+            if os.path.exists(canonical_exp_folder):
+                # Only ever delete a folder named exactly for this experiment
+                if os.path.basename(dst_real) != self.exp_id:
+                    raise RuntimeError(
+                        f"REFUSED: rmtree target '{dst_real}' is not this experiment's capsule."
+                    )
+                shutil.rmtree(canonical_exp_folder)
+            shutil.copytree(scratch_exp_folder, canonical_exp_folder)
+            print(f"[INFO] Results committed: {scratch_exp_folder} → {canonical_exp_folder}")
+
+        # Update csv_target to canonical location for final log message
+        csv_target = os.path.join(canonical_exp_folder, f"{self.exp_id}.csv")
+
+        # 3. Archive the EXACT source config into the capsule.
+        # Byte-faithful: the author's original file, NOT a yaml.dump
+        # re-serialization. A round-trip launders formatting that carries intent
+        # — underscored ints (1_000_000 -> 1000000), folded prose blocks ->
+        # escaped one-liners, real unicode dashes -> \uXXXX — and so the stored
+        # file would misrepresent "the exact config that ran." The Experiment ID
+        # is hashed from the PARSED dict, so byte-faithful archival changes no
+        # ID; the ID itself is recorded in the folder name and metadata_<ID>.json.
+        experiment_config_dest = os.path.join(canonical_exp_folder, "experiment_config.yaml")
+        self._archive_source_config(experiment_config_dest)
+
+        # 3.4 Embed the queries that ran — the selected engines' dialect SQL —
+        # into queries/ so a reader sees them without tracing fragments to
+        # source. A convenience copy; the Experiment ID hashes the full suite
+        # from source independently (see utils.common.copy_suite_queries).
+        copy_suite_queries(canonical_exp_folder)
+
+        # 3.5 Seal the capsule: aggregate hash over every file in the final
+        # canonical folder (the seal itself excluded). verify_capsule.py
+        # recomputes and compares — tamper evidence for published results.
+        # (Designed in docs/integrity_sealing.md; wiring completed here.)
+        seal = generate_integrity_seal(canonical_exp_folder)
+        with open(os.path.join(canonical_exp_folder, "integrity.seal"), "w") as f:
+            f.write(seal)
+
+        # 4. Archive Config registry
+        registry_path = os.path.join(CONFIG_ARCHIVE_DIR, f"config_{self.exp_id}.yaml")
+        os.makedirs(os.path.dirname(registry_path), exist_ok=True)
+        shutil.copy(ACTIVE_CONFIG_PATH, registry_path)
+
+        # 5. Archive copy in experiments/archive
+        filename = os.path.basename(self.target_yaml)
+        clean_name = filename if not filename.endswith(PROCESSED_SUFFIX) else filename[:-len(PROCESSED_SUFFIX)]
+        archive_dest = os.path.join(EXPERIMENTS_DIR, "archive", clean_name)
+        os.makedirs(os.path.dirname(archive_dest), exist_ok=True)
+        shutil.copy(os.environ["ACTIVE_CONFIG_PATH"], archive_dest)
+
+        print(f"[SUCCESS] Experiment {self.exp_id} finalized. Results at {csv_target}")
+        return is_semantically_valid
+
+    def _archive_source_config(self, dest_path: str) -> None:
+        """Write the captured source YAML (the exact bytes that were parsed and
+        hashed) verbatim into the capsule. Fail loud if it was never captured —
+        an empty/missing archived config is a silent provenance hole."""
+        if not self._source_yaml:
+            raise RuntimeError(
+                "REFUSED: no source config captured; cannot archive the capsule's "
+                "experiment_config.yaml. (Was the coordinator run via run()?)"
+            )
+        with open(dest_path, "w", encoding="utf-8") as f:
+            f.write(self._source_yaml)

sql_benchmarks/definitions.py

@@ -0,0 +1,115 @@
+import os
+from dagster import Definitions
+
+# 1. FACTORIES (Dynamic Lists)
+# We import the lists we explicitly built.
+from .assets.data_factory import data_assets
+from .assets.ingestion_factory import ingestion_assets
+from .assets.benchmark_factory import benchmark_assets
+from .assets.semantic_gate import get_semantic_gate_assets
+
+# 2. STATIC ASSETS (Explicit Import)
+# STOP using load_assets_from_modules here.
+# It creates duplicate keys because it scans imported variables.
+from .assets.reporting import performance_dashboard
+from .assets.maintenance import cleanup_staging_data
+from .assets.data_quality import quality_assets
+
+semantic_gate_assets = get_semantic_gate_assets(benchmark_assets)
+
+# 3. RESOURCES & INFRA
+from .resources.postgres import PostgresEngine
+from .resources.duckdb import DuckDBEngine
+from .resources.actian import ActianEngine
+from .resources.typedb_engine import TypeDBEngine
+from .resources.quack import QuackEngine
+from .constants import DATA_DIR
+from .jobs import benchmark_job
+# from .sensors import experiment_queue_sensor
+
+# 4. CONFIG
+pg_user = os.getenv("POSTGRES_USER", "postgres")
+pg_password = os.getenv("POSTGRES_PASSWORD", "password")
+pg_host = os.getenv("POSTGRES_HOST", "localhost")
+pg_port = os.getenv("POSTGRES_PORT", "5432")
+pg_db = os.getenv("POSTGRES_DB", "postgres")
+
+postgres_url = f"postgresql://{pg_user}:{pg_password}@{pg_host}:{pg_port}/{pg_db}"
+
+# 5. Definitions
+all_assets = [
+    *data_assets,
+    *ingestion_assets,
+    *benchmark_assets,
+    performance_dashboard,
+    cleanup_staging_data,
+    *quality_assets,
+    *semantic_gate_assets
+]
+
+typedb_address = os.getenv("TYPEDB_ADDRESS", "127.0.0.1:1729")
+
+# ---------------------------------------------------------------------------
+# Relation configs — keyed by base table name (no partition-key suffix).
+# Used by TypeDBEngine.bulk_load to dispatch to bulk_load_relation.
+# ---------------------------------------------------------------------------
+
+# Supply-chain hypergraph (3-way: supplier × buyer × product)
+SUPPLY_CHAIN_RELATION_CONFIGS = {
+    "supply_contract": {
+        "roles": {
+            "supplier_id": ["supplier", "supplier_role"],
+            "buyer_id":    ["buyer",    "buyer_role"],
+            "product_id":  ["product",  "product_role"],
+        },
+        "attributes": ["volume", "price_per_unit"],
+    }
+}
+
+# Recursive supply-graph (self-referential: company × company)
+# Both roles are played by the same entity type — TypeDB handles this natively.
+# "inference": "transitive" triggers _build_transitive_inference_schema after
+# the relation is loaded, adding a 'reachable' relation + recursive rule so that
+# q_transitive_closure.sql can query full reachability without explicit recursion.
+RECURSIVE_GRAPH_RELATION_CONFIGS = {
+    "supplies": {
+        "roles": {
+            "from_id": ["company", "seller_role"],
+            "to_id":   ["company", "buyer_role"],
+        },
+        "attributes": [],
+        "inference": "transitive",
+    }
+}
+
+# Active relation config — switch this to match the current experiment.
+# The TypeDBEngine is instantiated once at Dagster load time, so only one
+# experiment's relation config can be active per process.
+ACTIVE_RELATION_CONFIGS = RECURSIVE_GRAPH_RELATION_CONFIGS
+
+defs = Definitions(
+    assets=all_assets,
+    resources={
+        "postgres": PostgresEngine(connection_string=postgres_url),
+        "duckdb": DuckDBEngine(data_folder=os.path.join(DATA_DIR, "duckdb")),
+        "quack": QuackEngine(
+            data_folder=os.path.join(DATA_DIR, "quack"),
+            port=int(os.getenv("SB_QUACK_PORT", "9494")),
+            token=os.getenv("SB_QUACK_TOKEN", "sb-local-quack-token"),
+        ),
+        # Same protocol, server-side execution via remote.query() — own data
+        # folder and port so it never contends with the attach-mode engine.
+        "quack_pushdown": QuackEngine(
+            data_folder=os.path.join(DATA_DIR, "quack_pushdown"),
+            port=int(os.getenv("SB_QUACK_PUSHDOWN_PORT", "9495")),
+            token=os.getenv("SB_QUACK_TOKEN", "sb-local-quack-token"),
+            pushdown=True,
+        ),
+        "actian": ActianEngine(),
+        "typedb": TypeDBEngine(
+            address=typedb_address,
+            relation_configs=ACTIVE_RELATION_CONFIGS,
+        ),
+    },
+    jobs=[benchmark_job],
+)

sql_benchmarks/experiments/templates/experiment_template.yaml

@@ -0,0 +1,99 @@
+# =============================================================================
+# EXPERIMENT TEMPLATE — copy this, fill it in, drop the copy in experiments/queue/
+# =============================================================================
+# This is a complete, VALID experiment. Copy it, rename it (e.g.
+# my_experiment.yaml), edit the values, and place it in experiments/queue/ to run.
+#
+# The contract is enforced fail-fast at validation: sql_benchmarks/utils/schema.py
+# (Pydantic). If a field is wrong or missing, the run is rejected before anything
+# executes — there is no "partly valid" experiment.
+#
+# Two things worth knowing before you start:
+#   1. The Experiment ID is a SHA-256 of the PARSED config + the SQL + the
+#      measurement code. Change the question (engines, data, seed, suite, matrix)
+#      and the ID changes. Reformat the file (comments, spacing, quote style) and
+#      it does NOT — formatting is normalized out, like a code comment.
+#   2. Your file is archived into the capsule VERBATIM (byte-for-byte, comments
+#      and all). What you write here is what a reader sees in results/<ID>/.
+# =============================================================================
+
+meta:
+  # Human-readable identity. Optional, but always set them — they are how you and
+  # anyone else tell experiments apart. NOT hashed (the ID ignores meta), so you
+  # can fix a typo here without changing the ID.
+  name: "Short name of the experiment"
+  # The hypothesis, in prose. What question does this run answer?
+  description: >
+    One or more sentences stating the hypothesis and why this experiment tests
+    it. Folded blocks like this stay readable in the archived capsule.
+
+dataset:
+  # The data generator plugin. declarative_gen builds synthetic data from the
+  # column specs below. (For file-based data, use `paths:` instead of `tables:`.)
+  source: sql_benchmarks.plugins.data_sources.declarative_gen
+  # Base seed for synthetic data. HASHED — change it and the ID changes, so two
+  # experiments on different data can never share an identity. Defaults to 42.
+  seed: 42
+  tables:
+    # Table name — referenced by the SQL suite as {{ <name>_table }}.
+    my_table:
+      # Row count: a literal int, or a name from `definitions.rows` below so the
+      # matrix can sweep across scales.
+      rows: rows
+      columns:
+        # sequence: monotonic 1..N. Mark the primary key here.
+        - name: id
+          provider: sequence
+          primary_key: true
+        # choice: categorical values with weights (weights should sum to ~1.0).
+        # Use this to control selectivity — the fraction of rows a filter matches.
+        - name: category
+          provider: choice
+          options: ["a", "b", "c"]
+          weights: [0.1, 0.3, 0.6]
+        # random_int / random_float: numeric ranges via min/max.
+        - name: amount
+          provider: random_float
+          min: 0.0
+          max: 100.0
+        # text_concat: builds a string column (e.g. padding to force real scan
+        # cost). `source` references another column; `prefix` is a literal.
+        - name: payload
+          provider: text_concat
+          prefix: "PADDING_"
+          source: id
+      # OPTIONAL. Indexes built at ingestion, OUTSIDE the timed query loop.
+      # Row stores (Postgres) use these; columnar engines ignore them and rely on
+      # their own min-max zonemaps. Drop this block if you don't want indexes.
+      indexes:
+        - name: idx_my_table_category
+          columns: [category]
+
+# Named scales referenced by `rows:` above and swept by the matrix.
+definitions:
+  rows:
+    medium: 1_000_000
+    large: 10_000_000
+
+execution:
+  # The SQL suite to run: scripts/sql/<test_suite>/<dialect>/*.sql
+  test_suite: selectivity
+  # REQUIRED. Engines to run. Each maps to a SQL dialect directory.
+  # Available: duckdb, quack, quack_pushdown, postgres, actian, typedb.
+  engines:
+    - duckdb
+    - quack
+    - quack_pushdown
+    - postgres
+  # Replications per measured point. Raw per-rep timings are kept in the capsule.
+  replication: 5
+  # OPTIONAL per-engine tuning, namespaced by engine. Each engine sees only its
+  # own namespace. Delete if you don't need it.
+  # engine_params:
+  #   postgres: {work_mem: "64MB", random_page_cost: 1.1}
+  #   duckdb:   {threads: 4}
+  # The matrix: every combination becomes one isolated, cold-cache measurement.
+  matrix:
+    rows:
+      - medium
+      - large