sqlbenchdag 0.1.0__py3-none-any.whl

infrastructure/debug_aws.py

@@ -0,0 +1,40 @@
+import boto3
+import sys
+
+def check_instance_viability():
+    # Setup AWS client
+    # This will use the credentials from ~/.aws/credentials which the user already set up
+    ec2 = boto3.client('ec2', region_name='us-east-1')
+
+    print("--- DIAGNOSTIC: Checking t2.micro availability ---")
+    
+    try:
+        # 1. Describe Offerings to find valid AZs for t2.micro
+        response = ec2.describe_instance_type_offerings(
+            LocationType='availability-zone',
+            Filters=[{'Name': 'instance-type', 'Values': ['t2.micro']}]
+        )
+        
+        valid_azs = [o['Location'] for o in response['InstanceTypeOfferings']]
+        print(f"Valid AZs for t2.micro: {valid_azs}")
+        
+        if not valid_azs:
+            print("CRITICAL: t2.micro is NOT available in this region/account.")
+            return False
+
+        # 2. Check current default subnets
+        subnets = ec2.describe_subnets(Filters=[{'Name': 'default-for-az', 'Values': ['true']}])
+        print(f"Default Subnets Found: {len(subnets['Subnets'])}")
+        
+        for s in subnets['Subnets']:
+            print(f" - Subnet {s['SubnetId']} in {s['AvailabilityZone']}")
+            
+        print("--------------------------------------------------")
+        return True
+
+    except Exception as e:
+        print(f"AWS Error: {e}")
+        return False
+
+if __name__ == "__main__":
+    check_instance_viability()

scripts/dev/backfill_queries.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""
+Backfill queries/ into already-published capsules.
+
+For each capsule, embeds the SQL its engines actually ran (selected dialects,
+via utils.common.copy_suite_queries) and re-computes the integrity seal. This
+does NOT re-run any experiment and does NOT change the Experiment ID — the ID
+hashes the suite from source, not the capsule. After running this, the seal's
+sidecars are stale: re-timestamp with timestamp_capsule.py and re-sign the
+release tag (manual).
+
+Usage:  python scripts/dev/backfill_queries.py <id> [<id> ...]
+"""
+import os
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.insert(0, REPO_ROOT)
+
+from sql_benchmarks.utils.common import copy_suite_queries
+from sql_benchmarks.utils.hasher import generate_integrity_seal
+
+RESULTS = os.path.join(REPO_ROOT, "sql_benchmarks", "experiments", "results")
+
+
+def backfill(exp_id: str) -> None:
+    capsule = os.path.join(RESULTS, exp_id)
+    if not os.path.isdir(capsule):
+        print(f"{exp_id}: MISSING — skipped")
+        return
+    n = copy_suite_queries(capsule)
+    seal_path = os.path.join(capsule, "integrity.seal")
+    resealed = os.path.exists(seal_path)
+    if resealed:
+        with open(seal_path, "w") as f:
+            f.write(generate_integrity_seal(capsule))
+    print(f"{exp_id}: {n} query file(s) embedded; resealed={resealed}")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        sys.exit(__doc__)
+    for exp_id in sys.argv[1:]:
+        backfill(exp_id)

scripts/dev/timestamp_capsule.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""
+Timestamp a published capsule's seal with OpenTimestamps.
+
+Publication-time action (deliberate, network-required), distinct from sealing
+(automatic, offline, every run). Creates integrity.seal.ots — a trustless,
+Bitcoin-anchored proof that the seal hash existed by a point in time, so a
+result cannot be silently backdated or tampered-then-resealed.
+
+Usage:  python scripts/dev/timestamp_capsule.py <experiment_id> [<experiment_id> ...]
+Then later (after a Bitcoin block confirms, ~hours):
+        python scripts/dev/upgrade_capsule.py <experiment_id> [<experiment_id> ...]
+
+Requires: pip install opentimestamps-client
+"""
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+def timestamp_capsule(exp_id: str) -> bool:
+    seal = os.path.join(REPO_ROOT, "sql_benchmarks", "experiments", "results",
+                        exp_id, "integrity.seal")
+    if not os.path.exists(seal):
+        print(f"ERROR: {exp_id} has no integrity.seal — seal the capsule first "
+              f"(produced automatically when an experiment finalizes).")
+        return False
+    if not shutil.which("ots"):
+        print("ERROR: `ots` not found. pip install opentimestamps-client")
+        return False
+    proc = subprocess.run(["ots", "stamp", seal])
+    if proc.returncode != 0:
+        return False
+    print(f"\nStamped: {seal}.ots")
+    print("Commit it alongside the capsule. Run `ots upgrade` on it in a few "
+          "hours to finalize the Bitcoin attestation.")
+    return True
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="OpenTimestamp one or more capsule seals.")
+    parser.add_argument("ids", nargs="+", metavar="id", help="8-character Experiment ID(s)")
+    args = parser.parse_args()
+    results = [timestamp_capsule(exp_id) for exp_id in args.ids]
+    sys.exit(0 if all(results) else 1)

scripts/dev/upgrade_capsule.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""
+Finalize a capsule's OpenTimestamps proof (the second, later half of timestamping).
+
+`timestamp_capsule.py` submits the seal hash to calendar servers and writes a
+*pending* integrity.seal.ots. A few hours later, once a Bitcoin block has
+confirmed the calendar's commitment, run THIS to fetch the on-chain attestation
+and bake it into the .ots so it verifies offline forever — no trust in the
+calendar, no dependency on anyone.
+
+Usage:  python scripts/dev/upgrade_capsule.py <experiment_id> [<experiment_id> ...]
+Then commit the upgraded proof(s):
+        git commit -am "chore: finalize OTS attestation"
+
+Requires: pip install opentimestamps-client   (provides the `ots` CLI)
+
+Resolves paths relative to THIS file's repo, so run it from whichever checkout
+holds the capsule (normally your primary checkout after `git pull`).
+"""
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+def upgrade_capsule(exp_id: str) -> bool:
+    ots = os.path.join(REPO_ROOT, "sql_benchmarks", "experiments", "results",
+                       exp_id, "integrity.seal.ots")
+    if not os.path.exists(ots):
+        print(f"ERROR: {exp_id} has no integrity.seal.ots — stamp it first with "
+              f"`python scripts/dev/timestamp_capsule.py {exp_id}`.")
+        return False
+    if not shutil.which("ots"):
+        print("ERROR: `ots` not found. pip install opentimestamps-client")
+        return False
+
+    # `ots upgrade` rewrites the .ots in place when the Bitcoin attestation is
+    # ready, and leaves it untouched (exit 0) while still pending.
+    proc = subprocess.run(["ots", "upgrade", ots], capture_output=True, text=True)
+    out = (proc.stdout + proc.stderr).strip()
+    print(out or "(no output)")
+
+    if "Success" in out or "Got 1 attestation" in out:
+        print(f"\nFinalized: {ots}\nCommit it:  git commit -am 'chore: finalize OTS attestation for {exp_id}'")
+        return True
+    if "Pending" in out or "pending" in out:
+        print("\nStill pending — the Bitcoin block hasn't confirmed yet. "
+              "Try again in a few hours.")
+        return False
+    # Unknown state: surface it loudly rather than guess.
+    print("\nUnexpected `ots upgrade` output — inspect manually before committing.")
+    return proc.returncode == 0
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Finalize one or more capsules' OTS proofs.")
+    parser.add_argument("ids", nargs="+", metavar="id", help="8-character Experiment ID(s)")
+    args = parser.parse_args()
+    results = [upgrade_capsule(exp_id) for exp_id in args.ids]
+    sys.exit(0 if all(results) else 1)

scripts/dev/verify_capsule.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""
+Verify a published result capsule end to end:
+
+  1. INTEGRITY  — recompute the seal over the capsule's files and compare it
+                  to the stored integrity.seal (detects any byte change).
+  2. TIMESTAMP  — if integrity.seal.ots is present, verify the OpenTimestamps
+                  proof (the seal existed at/by a point in time; trustless,
+                  Bitcoin-anchored). Requires the `ots` client; skipped with a
+                  notice if unavailable.
+
+Authorship (who produced it) is verified separately via the signed git tag:
+  git verify-tag <tag>
+
+Usage:  python scripts/dev/verify_capsule.py <experiment_id>
+"""
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+
+# Run standalone from anywhere: put the repo root (3 levels up) on the path.
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.insert(0, REPO_ROOT)
+
+from sql_benchmarks.utils.hasher import generate_integrity_seal
+
+
+def verify_capsule(exp_id: str) -> bool:
+    results_dir = os.path.join(REPO_ROOT, "sql_benchmarks", "experiments", "results", exp_id)
+    if not os.path.isdir(results_dir):
+        print(f"ERROR: Result capsule for {exp_id} not found at {results_dir}")
+        return False
+
+    seal_path = os.path.join(results_dir, "integrity.seal")
+    if not os.path.exists(seal_path):
+        print(f"WARNING: Capsule {exp_id} is UNSEALED (no integrity.seal).")
+        return False
+
+    # 1. INTEGRITY
+    with open(seal_path) as f:
+        stored = f.read().strip()
+    computed = generate_integrity_seal(results_dir)
+    if computed != stored:
+        print(f"CRITICAL: integrity violation in {exp_id}!")
+        print(f"  stored:   {stored}")
+        print(f"  computed: {computed}")
+        return False
+    print(f"[1/2] INTEGRITY  OK — {exp_id} bytes match the seal.")
+
+    # 2. TIMESTAMP (optional sidecar)
+    ots_path = seal_path + ".ots"
+    if not os.path.exists(ots_path):
+        print(f"[2/2] TIMESTAMP  — no integrity.seal.ots (capsule not timestamped).")
+        return True
+    if not shutil.which("ots"):
+        print(f"[2/2] TIMESTAMP  — integrity.seal.ots present but `ots` client not "
+              f"installed; skipping (pip install opentimestamps-client).")
+        return True
+    proc = subprocess.run(["ots", "verify", ots_path], capture_output=True, text=True)
+    out = (proc.stdout + proc.stderr).lower()
+    if "pending" in out:
+        print(f"[2/2] TIMESTAMP  PENDING — calendars committed (submission time "
+              f"attested); run `ots upgrade {ots_path}` after the next Bitcoin "
+              f"block to finalize the trustless proof.")
+    elif "success" in out or "attests existence" in out or "bitcoin block" in out:
+        print(f"[2/2] TIMESTAMP  OK — seal anchored to the Bitcoin blockchain.")
+    else:
+        print(f"[2/2] TIMESTAMP  could not be verified:\n{proc.stdout}{proc.stderr}")
+        return False
+    return True
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Verify a result capsule (integrity + timestamp).")
+    parser.add_argument("id", help="8-character Experiment ID")
+    args = parser.parse_args()
+    sys.exit(0 if verify_capsule(args.id) else 1)

scripts/dev/verify_portability.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+import os
+import sys
+import subprocess
+import importlib
+from datetime import datetime
+
+def check_python_version():
+    print("[1/5] Checking Python Version...")
+    major, minor = sys.version_info[:2]
+    if major == 3 and minor >= 10:
+        print(f"      Python {major}.{minor} detected.")
+        return True
+    else:
+        print(f"      ERROR: Python 3.10+ required (Detected {major}.{minor})")
+        return False
+
+def check_dependencies():
+    print("[2/5] Checking Dependencies...")
+    required = ["dagster", "polars", "duckdb", "docker", "psutil", "yaml"]
+    missing = []
+    for lib in required:
+        try:
+            importlib.import_module(lib if lib != "yaml" else "yaml")
+            print(f"      {lib} is installed.")
+        except ImportError:
+            missing.append(lib)
+    
+    if missing:
+        print(f"      MISSING: {', '.join(missing)}")
+        return False
+    return True
+
+def check_docker():
+    print("[3/5] Checking Docker Daemon...")
+    try:
+        import docker
+        client = docker.from_env()
+        client.ping()
+        print("      Docker daemon is reachable.")
+        return True
+    except Exception as e:
+        print(f"      ERROR: Docker daemon not running or unreachable: {e}")
+        return False
+
+def check_filesystem():
+    print("[4/5] Checking Filesystem...")
+    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    required_dirs = ["data", "sql_benchmarks/experiments/queue", "dagster_home"]
+    
+    for d in required_dirs:
+        path = os.path.join(root, d)
+        if not os.path.exists(path):
+            try:
+                os.makedirs(path, exist_ok=True)
+                print(f"      Created {d}/")
+            except Exception as e:
+                print(f"      FAILED to create {d}/: {e}")
+                return False
+        else:
+            if os.access(path, os.W_OK):
+                print(f"      {d}/ is writable.")
+            else:
+                print(f"      {d}/ is NOT writable.")
+                return False
+    return True
+
+def check_dagster_definitions():
+    print("[5/5] Checking Dagster Definition Integrity...")
+    try:
+        # Set environment so definitions load correctly
+        root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        os.environ["DAGSTER_HOME"] = os.path.join(root, "dagster_home")
+        
+        # Import definitions to see if assets/factories fail to initialize
+        sys.path.append(root)
+        from sql_benchmarks.definitions import defs
+        print("      Dagster definitions loaded successfully.")
+        return True
+    except Exception as e:
+        print(f"      ERROR: Definitions failed to load: {e}")
+        return False
+
+def run_all():
+    print(f"--- SQL Benchmarks Portability Audit ({datetime.now().strftime('%Y-%m-%d %H:%M')}) ---")
+    results = [
+        check_python_version(),
+        check_dependencies(),
+        check_docker(),
+        check_filesystem(),
+        check_dagster_definitions()
+    ]
+    
+    print("-" * 60)
+    if all(results):
+        print("SUCCESS: Environment is ready for benchmarking.")
+        sys.exit(0)
+    else:
+        print("FAILURE: Please fix the issues above before running.")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    run_all()

scripts/tools/_plotlib.py

@@ -0,0 +1,43 @@
+"""Shared display metadata + paths for the figure tools (scripts/tools/plot_*.py).
+
+Design-once: an engine's label/color/marker, and the results path, live HERE.
+Every plot_*.py imports from this module, so adding an engine or restyling is a
+single edit — not a copy-paste across four scripts.
+"""
+import os
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+if REPO_ROOT not in sys.path:
+    sys.path.insert(0, REPO_ROOT)
+
+# Single source of truth for the results path — defined in the package, not redefined here.
+from sql_benchmarks.constants import RESULTS_DIR  # noqa: E402
+
+# engine -> (display label, hex color, marker). The ONE place to add/restyle an engine.
+# Keys track sql_benchmarks.constants.KNOWN_ENGINES.
+ENGINE_DISPLAY = {
+    "duckdb":         ("DuckDB in-process",             "#2f6f4f", "o"),
+    "quack":          ("Quack attach (ATTACH + USE)",   "#b3402a", "s"),
+    "quack_pushdown": ("Quack pushdown (remote.query)", "#3b6ea5", "^"),
+    "postgres":       ("PostgreSQL",                    "#7a5ea8", "D"),
+    "actian":         ("Actian Vector",                 "#c98a1b", "v"),
+    "typedb":         ("TypeDB",                         "#4c4c4c", "P"),
+}
+
+
+def style(engine: str):
+    """(label, color, marker) for an engine; falls back to the raw name + default marker."""
+    return ENGINE_DISPLAY.get(engine, (engine, None, "o"))
+
+
+def bench_string(env: dict, engines=()) -> str:
+    """One-line bench descriptor from a capsule's metadata environment.
+
+    Lists versions only for the engines the experiment actually used — never
+    hardcodes a single engine (so a Postgres-only figure won't claim 'DuckDB').
+    """
+    parts = [env.get("machine", ""), f"{env.get('cpu_count_logical', '?')} cores"]
+    versions = [f"{e} {env[e]}" for e in engines if env.get(e)]
+    tail = ", ".join(versions) if versions else (f"Python {env['python']}" if env.get("python") else "")
+    return " · ".join(p for p in (*parts, tail) if p)

scripts/tools/analyze_scaling.py

@@ -0,0 +1,39 @@
+"""
+Scaling-law analysis for a results capsule (CLI).
+
+Thin wrapper over sql_benchmarks.utils.scaling — the same code the reporting
+asset uses to write scaling.json into every capsule. Reads the sealed capsule,
+so the exponents are reproducible: anyone can rerun this against a published
+capsule and get the same numbers.
+
+Usage:
+    python scripts/tools/analyze_scaling.py <experiment_id>
+"""
+import os
+import sys
+
+# Allow running as a plain script from the repo root (python scripts/tools/...)
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from sql_benchmarks.constants import RESULTS_DIR
+from sql_benchmarks.utils.scaling import analyze_capsule
+
+
+def main():
+    if len(sys.argv) < 2:
+        sys.exit(__doc__)
+    exp_id = sys.argv[1]
+    report = analyze_capsule(RESULTS_DIR, exp_id)
+    if not report:
+        sys.exit(f"No row-scaled fragments with timings found for '{exp_id}'.")
+
+    print(f"Scaling analysis — capsule {exp_id}")
+    print(f"{'engine':<16} {'alpha':>7} {'R^2':>6} {'growth':>9}  regime")
+    print("-" * 56)
+    for engine in sorted(report):
+        r = report[engine]
+        print(f"{engine:<16} {r['alpha']:>7.3f} {r['r2']:>6.3f} {r['growth']:>8.1f}x  {r['regime']}")
+
+
+if __name__ == "__main__":
+    main()

scripts/tools/demo_breach_detection.py

@@ -0,0 +1,61 @@
+import os
+import shutil
+import time
+import yaml
+from sql_benchmarks.coordinator import ExperimentCoordinator
+from sql_benchmarks.constants import EXPERIMENTS_DIR
+
+def demonstrate_security():
+    """
+    Simulates a 'Blueberry Muffin' attack.
+    1. Starts a run.
+    2. While the coordinator is provisioning, we TAMPER with the code.
+    3. We verify the run FAILS to commit.
+    """
+    print("\n--- [DEMO] INITIATING BREACH TEST ---")
+    
+    # Setup a dummy experiment
+    exp_path = os.path.join(EXPERIMENTS_DIR, "queue", "breach_demo.yaml")
+    os.makedirs(os.path.dirname(exp_path), exist_ok=True)
+    with open(exp_path, 'w') as f:
+        yaml.dump({
+            "meta": {"experiment_id": "breach_demo"},
+            "execution": {"engines": ["duckdb"], "matrix": {"rows": [100]}}
+        }, f)
+
+    # We need to manually trigger the breach DURING the coordinator life-cycle
+    # For this demo, we will wrap the coordinator
+    coord = ExperimentCoordinator(exp_path, headless=True)
+    
+    # 1. INITIALIZE (Takes Snapshot)
+    print("[1/3] Starting Coordinator...")
+    
+    # We will simulate the 'Run' but inject a modification to a project file
+    # right after the harness provisions.
+    
+    target_file = "sql_benchmarks/constants.py"
+    original_content = open(target_file, "r").read()
+    
+    try:
+        # We start the run
+        print(f"[2/3] Tampering with {target_file} during run...")
+        with open(target_file, "a") as f:
+            f.write("\n# MALICIOUS INJECTION")
+            
+        print("[3/3] Attempting run with tampered code...")
+        success = coord.run()
+        
+        if not success:
+            print("\nSUCCESS: SYSTEM DETECTED THE BREACH!")
+            print("The results were DISCARDED and NOT committed to the repo.")
+        else:
+            print("\nFAILURE: SYSTEM ALLOWED THE BREACH TO COMMIT!")
+            
+    finally:
+        # Restore the file
+        with open(target_file, "w") as f:
+            f.write(original_content)
+        if os.path.exists(exp_path): os.remove(exp_path)
+
+if __name__ == "__main__":
+    demonstrate_security()

scripts/tools/extract_results.py

@@ -0,0 +1,57 @@
+import polars as pl
+import yaml
+import shutil
+import os
+import sys
+from datetime import datetime
+from dagster import DagsterInstance, DagsterEventType, DagsterRunStatus, RunsFilter
+
+# STRICTLY FORCE THE LOCATION THAT WORKED IN DEBUG
+os.environ["DAGSTER_HOME"] = os.path.expanduser("~/.dagster")
+
+# Import your constants for file paths
+from sql_benchmarks.constants import ACTIVE_CONFIG_PATH, CONFIG_ARCHIVE_DIR, RESULTS_DIR
+from sql_benchmarks.assets.reporting import parse_fragments_to_records
+
+def extract_and_snapshot():
+    # 1. READ TARGET ID
+    try:
+        with open(ACTIVE_CONFIG_PATH, "r") as f:
+            config = yaml.safe_load(f)
+        target_id = config.get("meta", {}).get("experiment_id")
+        print(f"Target Experiment ID: {target_id}")
+    except FileNotFoundError:
+        print(f"Config not found at {ACTIVE_CONFIG_PATH}")
+        return
+
+    # 2. EXTRACT RECORDS (Via Shared Logic)
+    print(f"Scanning fragments for experiment: {target_id}...")
+    records = parse_fragments_to_records(target_id)
+    
+    # 3. SAVE RESULTS
+    if records:
+        df = pl.DataFrame(records)
+        
+        # Deduplicate to keep the latest run of each asset (Logic reused from reporting.py implicitly via output parity)
+        df = df.unique(subset=["Asset", "System", "Rows"], keep="last").sort("Rows")
+        
+        # Define Paths
+        result_dir = os.path.join(RESULTS_DIR, target_id)
+        os.makedirs(result_dir, exist_ok=True)
+        os.makedirs(CONFIG_ARCHIVE_DIR, exist_ok=True)
+
+        csv_path = os.path.join(result_dir, f"results_{target_id}.csv")
+        registry_path = os.path.join(CONFIG_ARCHIVE_DIR, f"config_{target_id}.yaml")
+        
+        # Write
+        df.write_csv(csv_path)
+        shutil.copy(ACTIVE_CONFIG_PATH, registry_path)
+
+        print(f"\nSUCCESS!")
+        print(f"   Extracted {len(df)} rows")
+        print(f"   Saved to: {csv_path}")
+    else:
+        print(f"\nFAILURE: Scanned fragments but found 0 matches for ID '{target_id}'.")
+
+if __name__ == "__main__":
+    extract_and_snapshot()