sqlbenchdag 0.1.0__py3-none-any.whl

scripts/tools/plot_threads.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""
+Act II figure: the Quack pushdown server runs queries with FEWER effective
+threads than an in-process DuckDB connection. We sweep in-process `duckdb.threads`
+(1,2,4,8) and overlay pushdown (which ignores that knob — it runs in the server's
+own execution context) as a horizontal reference. Where the pushdown line meets
+the in-process curve is the server's *effective* thread count.
+
+Reads only the capsule's sealed fragments. Output: scratch/figures/threads_<id>.png
+
+Usage:  python scripts/tools/plot_threads.py [<experiment_id>]   (default 25b0e134)
+"""
+import glob
+import json
+import math
+import os
+import statistics
+import sys
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+from _plotlib import REPO_ROOT, RESULTS_DIR, style
+
+OUT = os.path.join(REPO_ROOT, "scratch", "figures")
+
+
+def load(exp_id):
+    s = {}
+    for p in glob.glob(os.path.join(RESULTS_DIR, exp_id, "fragments", "*.json")):
+        d = json.load(open(p))
+        par, m = d.get("parameters", {}), d.get("metrics", {})
+        raw = m.get("durations_raw") or (
+            [m["duration_seconds"]] if m.get("duration_seconds") is not None else [])
+        if not raw:
+            continue
+        th = par.get("threads") or par.get("duckdb.threads")
+        s.setdefault((d["meta"]["engine"], par.get("rows")), {}).setdefault(int(th), []).extend(raw)
+    return s
+
+
+def effective_threads(threads, durs, target):
+    """log-log interpolate the in-process curve to find threads where dur==target."""
+    xs = [math.log2(t) for t in threads]
+    ys = [math.log10(d) for d in durs]
+    ty = math.log10(target)
+    for i in range(len(xs) - 1):
+        if (ys[i] - ty) * (ys[i + 1] - ty) <= 0 and ys[i] != ys[i + 1]:
+            f = (ty - ys[i]) / (ys[i + 1] - ys[i])
+            return 2 ** (xs[i] + f * (xs[i + 1] - xs[i]))
+    return None
+
+
+def main(exp_id="25b0e134"):
+    s = load(exp_id)
+    os.makedirs(OUT, exist_ok=True)
+    rows_vals = sorted({r for (_, r) in s}, key=lambda x: int(x))
+    plt.figure(figsize=(7.5, 5))
+    colors = plt.cm.viridis([0.2, 0.7])
+    for c, rows in zip(colors, rows_vals):
+        dd = s.get(("duckdb", rows), {})
+        pd = s.get(("quack_pushdown", rows), {})
+        if not dd or not pd:
+            continue
+        threads = sorted(dd)
+        durs = [statistics.median(dd[t]) * 1000 for t in threads]
+        push = statistics.median([v for vals in pd.values() for v in vals]) * 1000
+        label_n = f"{int(rows):,} rows"
+        plt.plot(threads, durs, marker="o", color=c, label=f"{style('duckdb')[0]} — {label_n}")
+        plt.axhline(push, ls="--", color=c, alpha=0.8)
+        eff = effective_threads(threads, durs, push)
+        if eff:
+            plt.annotate(f"pushdown ≈ {eff:.1f} threads", xy=(eff, push), xytext=(eff, push * 1.35),
+                         color=c, fontsize=9, ha="center",
+                         arrowprops=dict(arrowstyle="->", color=c, alpha=0.7))
+        plt.text(8.1, push, f"pushdown {label_n}", va="center", fontsize=8, color=c)
+    plt.xscale("log", base=2)
+    plt.yscale("log")
+    plt.xticks([1, 2, 4, 8], ["1", "2", "4", "8"])
+    plt.xlabel("in-process DuckDB threads (of 8 cores)")
+    plt.ylabel("Median duration, ms (log scale)")
+    plt.title(f"Act II — the pushdown server runs at ~2–4 effective threads, not 8\n"
+              f"(dashed = pushdown, which ignores the thread knob; capsule {exp_id})")
+    plt.legend(loc="upper right", fontsize=8)
+    plt.grid(True, which="both", ls=":", alpha=0.4)
+    out = os.path.join(OUT, f"threads_{exp_id}.png")
+    plt.tight_layout()
+    plt.savefig(out, dpi=150)
+    plt.close()
+    print(f"wrote {out}")
+
+
+if __name__ == "__main__":
+    main(sys.argv[1] if len(sys.argv) > 1 else "25b0e134")

scripts/tools/regenerate_dashboard.py

@@ -0,0 +1,58 @@
+
+import sys
+import os
+import logging
+from dagster import build_asset_context
+
+# Setup paths (Robust)
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR = os.path.dirname(SCRIPT_DIR)
+if ROOT_DIR not in sys.path:
+    sys.path.append(ROOT_DIR)
+
+from sql_benchmarks.assets.reporting import performance_dashboard
+from sql_benchmarks.utils import common as common_utils
+
+import sql_benchmarks.assets.reporting
+print(f"[DEBUG] Loaded reporting from: {sql_benchmarks.assets.reporting.__file__}")
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='[%(levelname)s] %(message)s')
+
+def regenerate(exp_id):
+    print(f"Regenerating Dashboard for: {exp_id}")
+    
+    # 1. Mock Context for Experiment ID
+    import sql_benchmarks.assets.reporting
+    
+    def mock_load():
+        return {"meta": {"experiment_id": exp_id}}
+    
+    sql_benchmarks.assets.reporting.load_context = mock_load
+    
+    try:
+        # 2. Build Asset Context
+        context = build_asset_context()
+        # Redirect Dagster log to stdout
+        context.log.setLevel(logging.INFO)
+        context.log.addHandler(logging.StreamHandler(sys.stdout))
+        
+        # 3. Run Asset Logic
+        result = performance_dashboard(context)
+        
+        if result:
+            print("Dashboard Generated Successfully!")
+    except Exception as e:
+        print(f"Failed: {e}")
+        import traceback
+        traceback.print_exc()
+    finally:
+        # Restore (Optional, script is ending anyway)
+        pass
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python regenerate_dashboard.py <experiment_id>")
+        sys.exit(1)
+        
+    regenerate(sys.argv[1])

scripts/tools/verify_doc_claims.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""
+Verify that every capsule reference in the committed docs resolves to a real,
+git-tracked, sealed capsule. Catches the recurring error class: a doc citing a
+capsule ID that doesn't exist (fabricated or typo) or isn't committed (broken
+link) — e.g. the fabricated `3e2fe152` IDs an early article draft carried.
+
+Scans: README.md, AGENTS.md, and docs/**/*.md.
+Reference forms recognized (the house conventions for citing a capsule):
+  - path:    .../experiments/results/<8hex>/
+  - inline:  `<8hex>`   (backtick-quoted ID)
+
+Stdlib only — safe to run from a bare `python3` in a git pre-commit hook.
+
+Usage:  python scripts/tools/verify_doc_claims.py [--check]
+        --check : exit 1 if any reference is unresolved (for the hook / CI)
+"""
+import os
+import re
+import subprocess
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+RESULTS_REL = "sql_benchmarks/experiments/results"
+TOP_DOCS = ["README.md", "AGENTS.md"]
+PATH_RE = re.compile(r"results/([0-9a-f]{8})(?:/|\b)")
+TICK_RE = re.compile(r"`([0-9a-f]{8})`")
+
+# 8-hex tokens that are NOT capsule IDs (e.g. a short commit SHA cited in prose).
+# Add here if the verifier ever false-positives on a legitimate non-capsule hex.
+IGNORE = {
+    "48c92f31",  # illustrative example ID in README's Naming table — not a real capsule
+}
+
+
+def tracked_capsule_ids() -> set:
+    out = subprocess.run(
+        ["git", "ls-files", RESULTS_REL + "/"],
+        cwd=REPO_ROOT, capture_output=True, text=True,
+    )
+    ids = set()
+    for line in out.stdout.splitlines():
+        parts = line.split("/")
+        if len(parts) > 4:  # .../results/<ID>/<file>
+            ids.add(parts[3])
+    return ids
+
+
+def sealed(exp_id: str) -> bool:
+    return os.path.exists(os.path.join(REPO_ROOT, RESULTS_REL, exp_id, "integrity.seal"))
+
+
+def doc_files() -> list:
+    files = [os.path.join(REPO_ROOT, d) for d in TOP_DOCS
+             if os.path.exists(os.path.join(REPO_ROOT, d))]
+    for root, _, fs in os.walk(os.path.join(REPO_ROOT, "docs")):
+        files += [os.path.join(root, f) for f in fs if f.endswith(".md")]
+    return files
+
+
+def main() -> None:
+    check = "--check" in sys.argv[1:]
+    tracked = tracked_capsule_ids()
+    errors, ok = [], 0
+    for path in doc_files():
+        with open(path, encoding="utf-8") as f:
+            text = f.read()
+        refs = (set(PATH_RE.findall(text)) | set(TICK_RE.findall(text))) - IGNORE
+        rel = os.path.relpath(path, REPO_ROOT)
+        for rid in sorted(refs):
+            if rid not in tracked:
+                errors.append(f"{rel}: `{rid}` — referenced but NOT a git-tracked capsule "
+                              "(fabricated, typo, or uncommitted)")
+            elif not sealed(rid):
+                errors.append(f"{rel}: `{rid}` — tracked but has no integrity.seal")
+            else:
+                ok += 1
+    if errors:
+        print("DOC CLAIM CHECK — unresolved capsule references:")
+        for e in errors:
+            print("  ✗ " + e)
+    print(f"{ok} reference(s) resolve to sealed, tracked capsules; {len(errors)} problem(s).")
+    if check and errors:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

sql_benchmarks/__init__.py

@@ -0,0 +1,3 @@
+"""
+sql_benchmarks package.
+"""

sql_benchmarks/api/app.py

@@ -0,0 +1,30 @@
+from fastapi import FastAPI
+
+from .routers import catalog, experiments, recommend, results
+
+
+def create_app() -> FastAPI:
+    app = FastAPI(
+        title="SQL Benchmarks API",
+        description=(
+            "Ground-truth SQL performance data for researchers and AI agents. "
+            "Query pre-computed benchmark results across PostgreSQL, DuckDB, and Actian Vector, "
+            "or submit new experiments to run against the lab.\n\n"
+            "See [AGENTS.md](https://github.com/rctruta/sql-benchmarks-dagster/blob/main/AGENTS.md) for the agentic protocol."
+        ),
+        version="1.0.0",
+        docs_url="/docs",
+        openapi_url="/openapi.json",
+    )
+
+    app.include_router(catalog.router)
+    app.include_router(results.router)
+    app.include_router(recommend.router)
+    app.include_router(experiments.router)
+
+    @app.get("/health", tags=["meta"])
+    def health():
+        """Health check."""
+        return {"status": "ok"}
+
+    return app

sql_benchmarks/api/data/catalog_reader.py

@@ -0,0 +1,73 @@
+import os
+from typing import Dict, List
+
+from ...constants import KNOWN_ENGINES, SQL_DIR
+from ..models.catalog import CatalogEnginesResponse, CatalogSuitesResponse, EngineInfo, SuiteDetail
+
+
+class CatalogReader:
+    """Reads SQL catalog from the filesystem."""
+
+    def _suite_names(self) -> List[str]:
+        if not os.path.isdir(SQL_DIR):
+            return []
+        return [
+            d for d in os.listdir(SQL_DIR)
+            if os.path.isdir(os.path.join(SQL_DIR, d))
+        ]
+
+    def _suite_detail(self, suite_name: str) -> SuiteDetail:
+        suite_dir = os.path.join(SQL_DIR, suite_name)
+        sql_content: Dict[str, Dict[str, str]] = {}
+        engines_present = []
+
+        for engine in os.listdir(suite_dir):
+            engine_dir = os.path.join(suite_dir, engine)
+            if not os.path.isdir(engine_dir):
+                continue
+            sql_files = [f for f in os.listdir(engine_dir) if f.endswith(".sql")]
+            if not sql_files:
+                continue
+            engines_present.append(engine)
+            sql_content[engine] = {}
+            for fname in sql_files:
+                name = os.path.splitext(fname)[0]
+                with open(os.path.join(engine_dir, fname)) as f:
+                    sql_content[engine][name] = f.read()
+
+        benchmark_names = sorted({
+            name
+            for engine_files in sql_content.values()
+            for name in engine_files
+        })
+
+        return SuiteDetail(
+            name=suite_name,
+            engines=sorted(engines_present),
+            benchmark_names=benchmark_names,
+            sql_content=sql_content,
+        )
+
+    def get_suites_response(self) -> CatalogSuitesResponse:
+        suites = [self._suite_detail(s) for s in self._suite_names()]
+        return CatalogSuitesResponse(suites=suites)
+
+    def get_engines_response(self) -> CatalogEnginesResponse:
+        suite_names = self._suite_names()
+        engine_suites: Dict[str, List[str]] = {e: [] for e in KNOWN_ENGINES}
+
+        for suite_name in suite_names:
+            suite_dir = os.path.join(SQL_DIR, suite_name)
+            for engine in KNOWN_ENGINES:
+                engine_dir = os.path.join(suite_dir, engine)
+                if os.path.isdir(engine_dir) and any(
+                    f.endswith(".sql") for f in os.listdir(engine_dir)
+                ):
+                    engine_suites[engine].append(suite_name)
+
+        engines = [
+            EngineInfo(name=e, available_suites=sorted(suites))
+            for e, suites in engine_suites.items()
+            if suites
+        ]
+        return CatalogEnginesResponse(engines=engines)

sql_benchmarks/api/data/reader.py

@@ -0,0 +1,125 @@
+import glob
+import json
+import os
+from typing import Any, Dict, List, Optional
+
+import yaml
+
+from ...constants import CONFIG_ARCHIVE_DIR, EXPERIMENTS_DIR, RESULTS_DIR
+from ..models.results import (
+    ExperimentSummary,
+    Fragment,
+    FragmentMeta,
+    FragmentMetrics,
+)
+
+
+class ResultReader:
+    """Reads benchmark results from the filesystem. Stateless — reads on every call."""
+
+    def list_experiment_ids(self) -> List[str]:
+        if not os.path.isdir(RESULTS_DIR):
+            return []
+        return [
+            d for d in os.listdir(RESULTS_DIR)
+            if os.path.isdir(os.path.join(RESULTS_DIR, d))
+        ]
+
+    def get_metadata(self, exp_id: str) -> Optional[Dict]:
+        path = os.path.join(RESULTS_DIR, exp_id, f"metadata_{exp_id}.json")
+        if not os.path.exists(path):
+            return None
+        with open(path) as f:
+            return json.load(f)
+
+    def get_config(self, exp_id: str) -> Optional[Dict]:
+        path = os.path.join(CONFIG_ARCHIVE_DIR, f"config_{exp_id}.yaml")
+        if not os.path.exists(path):
+            return None
+        with open(path) as f:
+            return yaml.safe_load(f)
+
+    def get_fragments(self, exp_id: str) -> List[Fragment]:
+        pattern = os.path.join(RESULTS_DIR, exp_id, "fragments", "*.json")
+        fragments = []
+        for fpath in glob.glob(pattern):
+            try:
+                with open(fpath) as f:
+                    data = json.load(f)
+                if data.get("meta", {}).get("experiment_id") != exp_id:
+                    continue
+                meta = data.get("meta", {})
+                # Extract partition from filename: asset__PARTITION.json
+                filename = os.path.splitext(os.path.basename(fpath))[0]
+                partition = filename.split("__")[-1] if "__" in filename else "default"
+                fragments.append(Fragment(
+                    meta=FragmentMeta(
+                        timestamp=meta.get("timestamp", ""),
+                        experiment_id=meta.get("experiment_id", exp_id),
+                        dagster_run_id=meta.get("dagster_run_id", ""),
+                        engine=meta.get("engine", "unknown"),
+                        asset=meta.get("asset", ""),
+                        partition=partition,
+                    ),
+                    metrics=FragmentMetrics(
+                        duration_seconds=float(data.get("metrics", {}).get("duration_seconds", 0.0)),
+                        replication_factor=int(data.get("metrics", {}).get("replication_factor", 1)),
+                        durations_raw=data.get("metrics", {}).get("durations_raw"),
+                    ),
+                    parameters=data.get("parameters", {}),
+                ))
+            except Exception:
+                continue
+        return fragments
+
+    def has_csv(self, exp_id: str) -> bool:
+        return os.path.exists(os.path.join(RESULTS_DIR, exp_id, f"{exp_id}.csv"))
+
+    def has_dashboard(self, exp_id: str) -> bool:
+        return os.path.exists(os.path.join(RESULTS_DIR, exp_id, f"{exp_id}.html"))
+
+    def build_summary(self, exp_id: str) -> ExperimentSummary:
+        fragments = self.get_fragments(exp_id)
+        config = self.get_config(exp_id)
+        metadata = self.get_metadata(exp_id)
+
+        engines = list({f.meta.engine for f in fragments})
+        partitions = list({f.meta.partition for f in fragments})
+        suite = None
+        if config:
+            suite = config.get("execution", {}).get("test_suite")
+
+        return ExperimentSummary(
+            experiment_id=exp_id,
+            suite=suite,
+            engines=engines,
+            partition_count=len(partitions),
+            fragment_count=len(fragments),
+            has_csv=self.has_csv(exp_id),
+            has_dashboard=self.has_dashboard(exp_id),
+            created_at=metadata.get("timestamp") if metadata else None,
+        )
+
+    def filter_experiments(
+        self,
+        suite: Optional[str] = None,
+        engine: Optional[str] = None,
+    ) -> List[ExperimentSummary]:
+        summaries = []
+        for exp_id in self.list_experiment_ids():
+            summary = self.build_summary(exp_id)
+            if suite and summary.suite != suite:
+                continue
+            if engine and engine not in summary.engines:
+                continue
+            summaries.append(summary)
+        return summaries
+
+    def is_queued(self, exp_id: str) -> bool:
+        return os.path.exists(os.path.join(EXPERIMENTS_DIR, "queue", f"{exp_id}.yaml"))
+
+    def is_complete(self, exp_id: str) -> bool:
+        return os.path.exists(os.path.join(CONFIG_ARCHIVE_DIR, f"config_{exp_id}.yaml"))
+
+    def results_exist(self, exp_id: str) -> bool:
+        return os.path.isdir(os.path.join(RESULTS_DIR, exp_id))

sql_benchmarks/api/logic/comparator.py

@@ -0,0 +1,44 @@
+from typing import Optional
+
+from ..data.reader import ResultReader
+from ..logic.ranker import score_engines
+from ..models.results import CompareResult
+
+
+def compare_experiment(
+    exp_id: str,
+    reader: ResultReader,
+    partition: Optional[str] = None,
+) -> CompareResult:
+    fragments = reader.get_fragments(exp_id)
+    config = reader.get_config(exp_id)
+    suite = config.get("execution", {}).get("test_suite") if config else None
+
+    rankings = score_engines(fragments, partition_filter=partition)
+
+    if not rankings:
+        return CompareResult(
+            experiment_id=exp_id,
+            suite=suite,
+            partition=partition,
+            rankings=[],
+            winner="unknown",
+            speedup_vs_slowest=1.0,
+        )
+
+    winner = rankings[0]
+    slowest = rankings[-1]
+    speedup = (
+        slowest.mean_duration_seconds / winner.mean_duration_seconds
+        if winner.mean_duration_seconds > 0
+        else 1.0
+    )
+
+    return CompareResult(
+        experiment_id=exp_id,
+        suite=suite,
+        partition=partition,
+        rankings=rankings,
+        winner=winner.engine,
+        speedup_vs_slowest=round(speedup, 2),
+    )

sql_benchmarks/api/logic/ranker.py

@@ -0,0 +1,127 @@
+import statistics
+from typing import Dict, List, Optional
+
+from ..data.reader import ResultReader
+from ..models.recommend import RecommendResponse
+from ..models.results import EngineRanking, Fragment
+
+
+def score_engines(
+    fragments: List[Fragment],
+    partition_filter: Optional[str] = None,
+) -> List[EngineRanking]:
+    """
+    Groups fragments by engine, computes mean/median/p95 duration.
+    Returns rankings sorted ascending (rank 1 = fastest).
+    """
+    by_engine: Dict[str, List[float]] = {}
+    for f in fragments:
+        if partition_filter and partition_filter not in f.meta.partition:
+            continue
+        engine = f.meta.engine
+        by_engine.setdefault(engine, []).append(f.metrics.duration_seconds)
+
+    if not by_engine:
+        return []
+
+    stats = []
+    for engine, durations in by_engine.items():
+        sorted_d = sorted(durations)
+        n = len(sorted_d)
+        p95_idx = max(0, int(n * 0.95) - 1)
+        stats.append({
+            "engine": engine,
+            "mean": statistics.mean(sorted_d),
+            "median": statistics.median(sorted_d),
+            "p95": sorted_d[p95_idx],
+            "count": n,
+        })
+
+    stats.sort(key=lambda x: x["mean"])
+
+    return [
+        EngineRanking(
+            engine=s["engine"],
+            mean_duration_seconds=round(s["mean"], 6),
+            median_duration_seconds=round(s["median"], 6),
+            p95_duration_seconds=round(s["p95"], 6),
+            sample_count=s["count"],
+            rank=i + 1,
+        )
+        for i, s in enumerate(stats)
+    ]
+
+
+def recommend_engine(
+    reader: ResultReader,
+    suite: Optional[str],
+    scale: Optional[str],
+) -> RecommendResponse:
+    summaries = reader.filter_experiments(suite=suite)
+    exp_ids = [s.experiment_id for s in summaries]
+
+    all_fragments: List[Fragment] = []
+    for exp_id in exp_ids:
+        all_fragments.extend(reader.get_fragments(exp_id))
+
+    if scale:
+        all_fragments = [f for f in all_fragments if scale in f.meta.partition]
+
+    rankings = score_engines(all_fragments)
+    caveats = []
+    known_engines = {"postgres", "duckdb", "actian"}
+
+    if not rankings:
+        engines_with_data = set()
+    else:
+        engines_with_data = {r.engine for r in rankings}
+
+    for e in known_engines:
+        if e not in engines_with_data:
+            caveats.append(f"No data for '{e}'" + (f" in suite '{suite}'" if suite else ""))
+
+    if not rankings:
+        return RecommendResponse(
+            recommended_engine="unknown",
+            confidence="low",
+            reasoning="No benchmark data found for the requested filters.",
+            supporting_experiments=exp_ids,
+            engine_scores={},
+            caveats=caveats,
+        )
+
+    winner = rankings[0]
+    total_samples = sum(r.sample_count for r in rankings)
+
+    if len(rankings) > 1:
+        second = rankings[1]
+        speedup = second.mean_duration_seconds / winner.mean_duration_seconds
+    else:
+        speedup = 1.0
+
+    if winner.sample_count >= 10 and speedup >= 1.20:
+        confidence = "high"
+    elif winner.sample_count >= 3:
+        confidence = "medium"
+    else:
+        confidence = "low"
+
+    suite_label = f"suite '{suite}'" if suite else "available suites"
+    scale_label = f" at scale '{scale}'" if scale else ""
+    reasoning = (
+        f"Based on {total_samples} samples across {suite_label}{scale_label}, "
+        f"{winner.engine} is fastest with mean {winner.mean_duration_seconds:.4f}s"
+    )
+    if len(rankings) > 1:
+        reasoning += f" vs {rankings[1].engine} at {rankings[1].mean_duration_seconds:.4f}s ({speedup:.1f}x speedup)."
+    else:
+        reasoning += "."
+
+    return RecommendResponse(
+        recommended_engine=winner.engine,
+        confidence=confidence,
+        reasoning=reasoning,
+        supporting_experiments=exp_ids,
+        engine_scores={r.engine: r.mean_duration_seconds for r in rankings},
+        caveats=caveats,
+    )

sql_benchmarks/api/models/catalog.py

@@ -0,0 +1,22 @@
+from pydantic import BaseModel
+from typing import Dict, List
+
+
+class EngineInfo(BaseModel):
+    name: str
+    available_suites: List[str]
+
+
+class SuiteDetail(BaseModel):
+    name: str
+    engines: List[str]
+    benchmark_names: List[str]
+    sql_content: Dict[str, Dict[str, str]]  # {engine: {benchmark_name: sql_text}}
+
+
+class CatalogEnginesResponse(BaseModel):
+    engines: List[EngineInfo]
+
+
+class CatalogSuitesResponse(BaseModel):
+    suites: List[SuiteDetail]