sqlquality 0.2.0__py3-none-any.whl

sqlquality/__init__.py

@@ -0,0 +1,3 @@
+"""sqlquality — measure dbt model performance and complexity."""
+
+__version__ = "0.2.0"

sqlquality/adapters/__init__.py

@@ -0,0 +1,20 @@
+"""Perf adapter registry."""
+
+from __future__ import annotations
+
+from sqlquality.adapters.base import PerfAdapter
+from sqlquality.adapters.postgres import PostgresAdapter
+from sqlquality.adapters.redshift import RedshiftAdapter
+
+_ADAPTERS: dict[str, type[PerfAdapter]] = {
+    "postgres": PostgresAdapter,
+    "redshift": RedshiftAdapter,
+}
+
+
+def get_adapter(engine: str) -> PerfAdapter:
+    """Return the perf adapter for an engine, or raise ValueError."""
+    try:
+        return _ADAPTERS[engine]()
+    except KeyError:
+        raise ValueError(f"No perf adapter for dialect '{engine}'")

sqlquality/adapters/base.py

@@ -0,0 +1,21 @@
+"""PerfAdapter interface — one per SQL dialect/engine."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from sqlquality.models import Finding
+
+
+class PerfAdapter(ABC):
+    """Per-engine performance analyzer (static + EXPLAIN-plan)."""
+
+    engine: str
+
+    @abstractmethod
+    def static_findings(self, sql: str) -> list[Finding]:
+        """Static anti-pattern findings from the SQL text."""
+
+    @abstractmethod
+    def plan_findings(self, explain_text: str) -> list[Finding]:
+        """Findings from a captured EXPLAIN output (raw file text; format per engine)."""

sqlquality/adapters/postgres.py

@@ -0,0 +1,60 @@
+"""Postgres performance adapter: static anti-patterns + EXPLAIN-JSON parsing."""
+
+from __future__ import annotations
+
+from sqlquality.adapters.base import PerfAdapter
+from sqlquality.antipatterns import antipattern_findings
+from sqlquality.models import Finding, Severity
+
+
+def parse_pg_plan(plan: object) -> list[Finding]:
+    """Findings from Postgres EXPLAIN (FORMAT JSON) output."""
+    findings: list[Finding] = []
+
+    def walk(node: dict) -> None:
+        if node.get("Node Type") == "Seq Scan":
+            rel = node.get("Relation Name", "?")
+            findings.append(
+                Finding(
+                    "PG001",
+                    f"Seq Scan on {rel} — consider an index if the filter is selective.",
+                    0,
+                    Severity.WARNING,
+                    False,
+                )
+            )
+        if node.get("Node Type") == "Sort" and node.get("Sort Space Type") == "Disk":
+            findings.append(
+                Finding(
+                    "PG002",
+                    "Sort spilled to disk (external merge) — raise work_mem or reduce the sorted set.",
+                    0,
+                    Severity.WARNING,
+                    False,
+                )
+            )
+        for child in node.get("Plans") or []:
+            walk(child)
+
+    items = plan if isinstance(plan, list) else [plan]
+    for item in items:
+        node = item.get("Plan", item) if isinstance(item, dict) else {}
+        if node:
+            walk(node)
+    return findings
+
+
+class PostgresAdapter(PerfAdapter):
+    engine = "postgres"
+
+    def static_findings(self, sql: str) -> list[Finding]:
+        return antipattern_findings(sql, "postgres")
+
+    def plan_findings(self, explain_text: str) -> list[Finding]:
+        import json
+
+        try:
+            plan = json.loads(explain_text)
+        except json.JSONDecodeError as exc:
+            raise ValueError(f"Postgres EXPLAIN must be FORMAT JSON: {exc}") from exc
+        return parse_pg_plan(plan)

sqlquality/adapters/redshift.py

@@ -0,0 +1,46 @@
+"""Redshift performance adapter: anti-patterns + dist/sort inference + EXPLAIN markers."""
+
+from __future__ import annotations
+
+from sqlquality.adapters.base import PerfAdapter
+from sqlquality.antipatterns import antipattern_findings
+from sqlquality.keys import dist_sort_findings
+from sqlquality.models import Finding, Severity
+
+_MARKERS = [
+    (
+        "DS_BCAST_INNER",
+        "RS010",
+        "Broadcast of inner table (DS_BCAST_INNER) — tables not joined on their DISTKEYs.",
+    ),
+    (
+        "DS_DIST_BOTH",
+        "RS011",
+        "Both sides redistributed (DS_DIST_BOTH) — the heaviest redistribution; align DISTKEYs on the join key.",
+    ),
+    (
+        "DS_DIST_ALL_INNER",
+        "RS012",
+        "Serial execution (DS_DIST_ALL_INNER) — inner table sent to a single slice.",
+    ),
+    ("Nested Loop", "RS013", "Nested Loop join — usually a missing join condition / cross join."),
+]
+
+
+def parse_redshift_plan(explain_text: str) -> list[Finding]:
+    """Findings from a Redshift EXPLAIN text plan (redistribution / join markers)."""
+    findings: list[Finding] = []
+    for marker, code, message in _MARKERS:
+        if marker in explain_text:
+            findings.append(Finding(code, message, 0, Severity.WARNING, False))
+    return findings
+
+
+class RedshiftAdapter(PerfAdapter):
+    engine = "redshift"
+
+    def static_findings(self, sql: str) -> list[Finding]:
+        return antipattern_findings(sql, "redshift") + dist_sort_findings(sql, "redshift")
+
+    def plan_findings(self, explain_text: str) -> list[Finding]:
+        return parse_redshift_plan(explain_text)

sqlquality/antipatterns.py

@@ -0,0 +1,174 @@
+"""Dialect-agnostic static SQL anti-pattern detectors (SQLGlot)."""
+
+from __future__ import annotations
+
+from sqlglot import exp
+
+from sqlquality.models import Finding, Severity
+from sqlquality.sqlast import SqlParseError, parse
+
+
+def _is_star_projection(projection: exp.Expression) -> bool:
+    return isinstance(projection, exp.Star) or (
+        isinstance(projection, exp.Column) and isinstance(projection.this, exp.Star)
+    )
+
+
+def _within_exists(node: exp.Expression) -> bool:
+    """True if ``node`` sits inside an ``EXISTS (...)`` subquery."""
+    parent = node.parent
+    while parent is not None:
+        if isinstance(parent, exp.Exists):
+            return True
+        parent = parent.parent
+    return False
+
+
+def _is_cte_closer(select: exp.Select, cte_names: set[str]) -> bool:
+    """True for the idiomatic dbt closer ``select * from <cte>`` (sole star, no joins)."""
+    if len(select.expressions) != 1 or not isinstance(select.expressions[0], exp.Star):
+        return False
+    if select.args.get("joins"):
+        return False
+    from_clause = select.args.get("from_")
+    if from_clause is None:
+        return False
+    table = from_clause.this
+    return isinstance(table, exp.Table) and table.name.lower() in cte_names
+
+
+def _has_select_star(tree: exp.Expression) -> bool:
+    cte_names = {cte.alias.lower() for cte in tree.find_all(exp.CTE)}
+    for select in tree.find_all(exp.Select):
+        if not any(_is_star_projection(p) for p in select.expressions):
+            continue
+        # (a) `EXISTS (SELECT * ...)` only probes for row existence — semantically free.
+        if _within_exists(select):
+            continue
+        # (b) idiomatic dbt closer `select * from final` where `final` is a local CTE.
+        if _is_cte_closer(select, cte_names):
+            continue
+        return True
+    return False
+
+
+def _is_constant_true(on: exp.Expression) -> bool:
+    """True for a constant-true join condition: ``ON TRUE`` or ``ON 1=1``."""
+    if isinstance(on, exp.Boolean) and on.this is True:
+        return True
+    if isinstance(on, exp.EQ):
+        left, right = on.this, on.expression
+        if isinstance(left, exp.Literal) and isinstance(right, exp.Literal) and left == right:
+            return True
+    return False
+
+
+def _nearest_select(node: exp.Expression) -> exp.Select | None:
+    parent = node.parent
+    while parent is not None:
+        if isinstance(parent, exp.Select):
+            return parent
+        parent = parent.parent
+    return None
+
+
+def _where_joins_relation(select: exp.Select, relation: str) -> bool:
+    """True if ``select``'s own WHERE equates ``relation`` to a *different* relation.
+
+    Scoped deliberately: only the WHERE of the join's owning SELECT is consulted, and
+    only equalities whose nearest SELECT ancestor is that same SELECT count — so a
+    predicate nested in a subquery/EXISTS cannot exonerate an outer comma join, and an
+    outer predicate cannot exonerate a comma join buried in a CTE.
+    """
+    where = select.args.get("where")
+    if where is None:
+        return False
+    for eq in where.find_all(exp.EQ):
+        if _nearest_select(eq) is not select:
+            continue
+        left, right = eq.this, eq.expression
+        if not (isinstance(left, exp.Column) and isinstance(right, exp.Column)):
+            continue
+        left_tbl, right_tbl = left.table, right.table
+        if left_tbl == relation and right_tbl and right_tbl != relation:
+            return True
+        if right_tbl == relation and left_tbl and left_tbl != relation:
+            return True
+    return False
+
+
+def _has_cartesian_join(tree: exp.Expression) -> bool:
+    for join in tree.find_all(exp.Join):
+        if (join.args.get("method") or "").upper() == "NATURAL":
+            continue
+        # LATERAL joins correlate through the lateral body, not an ON clause; the
+        # `... ON TRUE` idiom is expected and must not be flagged.
+        if isinstance(join.this, exp.Lateral):
+            continue
+        if join.args.get("using"):
+            continue
+        on = join.args.get("on")
+        if on is not None:
+            # False negative: a constant-true ON is a disguised cross join.
+            if _is_constant_true(on):
+                return True
+            continue
+        # Explicit CROSS JOIN is always flagged (behavior kept as-is).
+        if (join.args.get("kind") or "").upper() == "CROSS":
+            return True
+        # Old-style comma join: not cartesian if this SELECT's own WHERE joins it.
+        owner = join.parent
+        relation = join.this.alias_or_name if isinstance(join.this, exp.Expression) else ""
+        if isinstance(owner, exp.Select) and relation and _where_joins_relation(owner, relation):
+            continue
+        return True
+    return False
+
+
+def _has_leading_wildcard_like(tree: exp.Expression) -> bool:
+    for node in tree.find_all(exp.Like, exp.ILike):
+        pattern = node.args.get("expression")
+        if isinstance(pattern, exp.Literal) and pattern.is_string and pattern.this.startswith("%"):
+            return True
+    return False
+
+
+def antipattern_findings(sql: str, dialect: str) -> list[Finding]:
+    """Static anti-pattern findings for one SQL statement."""
+    try:
+        tree = parse(sql, dialect)
+    except SqlParseError as exc:
+        return [Finding("SQ000", f"Unparseable SQL: {exc}", 0, Severity.ERROR, False)]
+
+    findings: list[Finding] = []
+    if _has_select_star(tree):
+        findings.append(
+            Finding(
+                "SQ001",
+                "SELECT * projects an unknown/wide column set; list columns explicitly.",
+                0,
+                Severity.WARNING,
+                False,
+            )
+        )
+    if _has_cartesian_join(tree):
+        findings.append(
+            Finding(
+                "SQ002",
+                "Cartesian/cross join without an ON/USING condition.",
+                0,
+                Severity.WARNING,
+                False,
+            )
+        )
+    if _has_leading_wildcard_like(tree):
+        findings.append(
+            Finding(
+                "SQ003",
+                "Leading-wildcard LIKE ('%...') is non-sargable and cannot use an index.",
+                0,
+                Severity.WARNING,
+                False,
+            )
+        )
+    return findings

sqlquality/changeset.py

@@ -0,0 +1,93 @@
+"""Turn dbt `state:modified` selection into a changed-model ChangeSet."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+from sqlquality.dbtproject import DbtProject
+
+
+class ChangeSetError(RuntimeError):
+    """Raised when the `dbt ls` invocation fails."""
+
+
+@dataclass(frozen=True)
+class ChangeSet:
+    changed: list[str]
+    neighbors: list[str]
+
+    @property
+    def analysis_set(self) -> list[str]:
+        return sorted(set(self.changed) | set(self.neighbors))
+
+
+def parse_state_modified(stdout: str) -> list[str]:
+    """Extract model unique_ids from `dbt ls --output json` JSONL output."""
+    ids: set[str] = set()
+    for line in stdout.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            obj = json.loads(line)
+        except json.JSONDecodeError:
+            continue  # dbt can interleave non-JSON log lines
+        if isinstance(obj, dict) and obj.get("resource_type") == "model":
+            uid = obj.get("unique_id")
+            if uid:
+                ids.add(uid)
+    return sorted(ids)
+
+
+def compute_changeset(project: DbtProject, ls_stdout: str) -> ChangeSet:
+    """Changed models (from `dbt ls`) plus their 1-hop model neighbors."""
+    models = set(project.model_ids())
+    changed = [uid for uid in parse_state_modified(ls_stdout) if uid in models]
+    changed_set = set(changed)
+    neighbors: set[str] = set()
+    for uid in changed:
+        neighbors.update(project.model_parents(uid))
+        neighbors.update(project.model_children(uid))
+    neighbors -= changed_set
+    return ChangeSet(changed=changed, neighbors=sorted(neighbors))
+
+
+def run_state_modified(project_dir: str | Path, state_dir: str | Path, dbt: str = "dbt") -> str:
+    """Run `dbt ls --select state:modified ... --output json` and return stdout."""
+    # --no-write-json: `dbt ls` would otherwise rewrite the candidate's
+    # target/manifest.json without compiled_code, self-neutralizing the gate on
+    # the next run. --state is resolved absolute because the subprocess runs
+    # with cwd=project_dir, but the CLI resolves --state against the real cwd.
+    cmd = [
+        dbt,
+        "ls",
+        "--no-write-json",
+        "--select",
+        "state:modified",
+        "--state",
+        str(Path(state_dir).resolve()),
+        "--resource-type",
+        "model",
+        "--output",
+        "json",
+    ]
+    try:
+        result = subprocess.run(
+            cmd, cwd=Path(project_dir), capture_output=True, text=True, timeout=600
+        )
+    except FileNotFoundError as exc:
+        raise ChangeSetError(
+            f"dbt executable '{dbt}' not found on PATH — install dbt or pass --dbt"
+        ) from exc
+    except subprocess.TimeoutExpired as exc:
+        raise ChangeSetError(f"`dbt ls` timed out after {exc.timeout:.0f}s") from exc
+    if result.returncode != 0:
+        # dbt logs errors to stdout, not stderr; fall back to the stdout tail.
+        detail = result.stderr.strip()
+        if not detail:
+            detail = "\n".join(result.stdout.strip().splitlines()[-20:])
+        raise ChangeSetError(f"`dbt ls` failed (exit {result.returncode}): {detail}")
+    return result.stdout