sqlquality 0.2.0__py3-none-any.whl

sqlquality/dbtproject.py

@@ -0,0 +1,126 @@
+"""Read a dbt project's manifest.json (schema v12) as a model graph."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from sqlquality.models import DagFacts
+
+
+class DbtProjectError(ValueError):
+    """Raised when the manifest is malformed or a node is missing/uncompiled."""
+
+
+@dataclass(frozen=True)
+class ModelNode:
+    unique_id: str
+    name: str
+    resource_type: str
+    materialized: str | None
+    compiled_code: str | None
+    relation_name: str | None
+    depends_on: list[str]
+    config: dict
+
+
+class DbtProject:
+    """A loaded dbt manifest, indexed for model-graph queries."""
+
+    def __init__(self, manifest: dict) -> None:
+        self._manifest = manifest
+        self._nodes: dict = manifest.get("nodes", {})
+        self._parent_map: dict = manifest.get("parent_map", {})
+        self._child_map: dict = manifest.get("child_map", {})
+        self._depth_cache: dict[str, int] = {}
+
+    @classmethod
+    def from_manifest(cls, manifest: dict) -> "DbtProject":
+        return cls(manifest)
+
+    @classmethod
+    def from_path(cls, manifest_path: str | Path) -> "DbtProject":
+        path = Path(manifest_path)
+        try:
+            manifest = json.loads(path.read_text())
+        except (OSError, json.JSONDecodeError) as exc:
+            raise DbtProjectError(f"Could not read manifest {path}: {exc}") from exc
+        return cls(manifest)
+
+    def adapter_type(self) -> str:
+        return self._manifest.get("metadata", {}).get("adapter_type", "")
+
+    def schema_version(self) -> str:
+        return self._manifest.get("metadata", {}).get("dbt_schema_version", "")
+
+    def _is_model(self, uid: str) -> bool:
+        node = self._nodes.get(uid)
+        return node is not None and node.get("resource_type") == "model"
+
+    def model_ids(self) -> list[str]:
+        return sorted(uid for uid in self._nodes if self._is_model(uid))
+
+    def node(self, uid: str) -> ModelNode:
+        raw = self._nodes.get(uid)
+        if raw is None:
+            raise DbtProjectError(f"No such node: {uid}")
+        config = raw.get("config") or {}
+        return ModelNode(
+            unique_id=uid,
+            name=raw.get("name", ""),
+            resource_type=raw.get("resource_type", ""),
+            materialized=config.get("materialized"),
+            compiled_code=raw.get("compiled_code"),
+            relation_name=raw.get("relation_name"),
+            depends_on=list(raw.get("depends_on", {}).get("nodes", [])),
+            config=config,
+        )
+
+    def model_parents(self, uid: str) -> list[str]:
+        return sorted(p for p in self._parent_map.get(uid, []) if self._is_model(p))
+
+    def model_children(self, uid: str) -> list[str]:
+        return sorted(c for c in self._child_map.get(uid, []) if self._is_model(c))
+
+    def compiled_sql(self, uid: str) -> str:
+        node = self.node(uid)
+        if not node.compiled_code:
+            raise DbtProjectError(f"{uid} has no compiled_code — run `dbt compile` first")
+        return node.compiled_code
+
+    def dag_facts(self, uid: str) -> DagFacts:
+        return DagFacts(
+            fan_in=len(self.model_parents(uid)),
+            fan_out=len(self.model_children(uid)),
+            lineage_depth=self._lineage_depth(uid),
+        )
+
+    def _lineage_depth(self, uid: str) -> int:
+        if uid in self._depth_cache:
+            return self._depth_cache[uid]
+        # Iterative post-order DFS: a recursive walk overflows the stack on very
+        # deep chains and never terminates on cycles. `on_stack` marks nodes whose
+        # depth is still being computed; a parent still on the stack closes a cycle
+        # and contributes depth 0, keeping the result finite.
+        stack: list[str] = [uid]
+        on_stack: set[str] = set()
+        while stack:
+            node = stack[-1]
+            if node in self._depth_cache:
+                stack.pop()
+                continue
+            on_stack.add(node)
+            parents = self.model_parents(node)
+            pending = [p for p in parents if p not in self._depth_cache and p not in on_stack]
+            if pending:
+                stack.extend(pending)
+                continue
+            depth = 1 + max(
+                (self._depth_cache[p] for p in parents if p in self._depth_cache),
+                default=0,
+            )
+            self._depth_cache[node] = depth
+            on_stack.discard(node)
+            stack.pop()
+        return self._depth_cache[uid]

sqlquality/delta.py

@@ -0,0 +1,76 @@
+"""Compute per-model complexity deltas between a baseline and candidate project."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from sqlquality.complexity import ComplexityEngine
+from sqlquality.dbtproject import DbtProject, DbtProjectError
+from sqlquality.sqlast import SqlParseError, analyze_sql
+
+
+@dataclass(frozen=True)
+class ModelDelta:
+    unique_id: str
+    baseline: float
+    candidate: float
+    delta: float
+    is_new: bool
+
+
+_ENGINE = ComplexityEngine()
+
+
+def _composite(project: DbtProject, uid: str, dialect: str) -> float | None:
+    """Complexity composite for one model, or None if SQL missing/unparseable."""
+    try:
+        sql = project.compiled_sql(uid)
+        metrics = analyze_sql(sql, dialect)
+    except (DbtProjectError, SqlParseError):
+        return None
+    return _ENGINE.score(metrics, project.dag_facts(uid)).composite
+
+
+def compute_deltas(
+    baseline: DbtProject | None,
+    candidate: DbtProject,
+    changed_ids: list[str],
+    dialect: str,
+) -> tuple[list[ModelDelta], list[tuple[str, str]]]:
+    """Score each changed model on candidate and baseline; return deltas + skips."""
+    deltas: list[ModelDelta] = []
+    skipped: list[tuple[str, str]] = []
+    baseline_model_ids = set() if baseline is None else set(baseline.model_ids())
+    for uid in changed_ids:
+        cand = _composite(candidate, uid, dialect)
+        if cand is None:
+            skipped.append((uid, "no compiled SQL or unparseable in candidate"))
+            continue
+        if baseline is None:
+            is_new = True
+            base_value = 0.0
+        else:
+            base = _composite(baseline, uid, dialect)
+            if base is None:
+                # A node absent from the baseline is genuinely net-new. A node
+                # *present* in the baseline but uncompiled/unparseable must NOT be
+                # exempted as "new" — that would let a real regression through the
+                # gate. Skip it instead of emitting a misleading delta.
+                if uid in baseline_model_ids:
+                    skipped.append((uid, "baseline present but unscoreable"))
+                    continue
+                is_new = True
+                base_value = 0.0
+            else:
+                is_new = False
+                base_value = base
+        deltas.append(
+            ModelDelta(
+                unique_id=uid,
+                baseline=base_value,
+                candidate=cand,
+                delta=round(cand - base_value, 1),
+                is_new=is_new,
+            )
+        )
+    return deltas, skipped

sqlquality/dialects.py

@@ -0,0 +1,23 @@
+"""Validation of SQL dialect names against SQLGlot's registry."""
+
+from __future__ import annotations
+
+import sqlglot
+
+#: A short, friendly sample of dialects to suggest when an unknown name is given.
+KNOWN_DIALECTS: tuple[str, ...] = ("postgres", "redshift", "snowflake", "bigquery", "duckdb")
+
+
+def validate_dialect(name: str) -> str:
+    """Return the normalized dialect name, or raise ``ValueError`` if unknown.
+
+    The name is lowercased and stripped, then checked against SQLGlot's dialect
+    registry via :meth:`sqlglot.Dialect.get`. An empty/blank name is rejected too,
+    since it is not a meaningful dialect to validate.
+    """
+    normalized = name.strip().lower()
+    if not normalized or sqlglot.Dialect.get(normalized) is None:
+        raise ValueError(
+            f"Unknown SQL dialect {name!r}. Known dialects include: {', '.join(KNOWN_DIALECTS)}."
+        )
+    return normalized

sqlquality/gate.py

@@ -0,0 +1,37 @@
+"""Apply gate config to model deltas to produce a pass/fail verdict."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from sqlquality.config import Config
+from sqlquality.delta import ModelDelta
+
+
+@dataclass(frozen=True)
+class GateReport:
+    deltas: list[ModelDelta]
+    regressions: list[str]
+    passed: bool
+    mode: str = "warn"
+
+    @property
+    def warned(self) -> bool:
+        """True when warn mode let regressions through (passed, but not clean)."""
+        return self.mode == "warn" and bool(self.regressions)
+
+
+def evaluate_gate(deltas: list[ModelDelta], config: Config) -> GateReport:
+    """Flag regressions (over threshold, not waived); fail only in 'fail' mode."""
+    threshold = config.gate.max_complexity_increase
+    waivers = set(config.waivers)
+    # A brand-new model (no baseline) has delta == its full composite; a *delta*
+    # gate does not treat net-new surface as a regression. Absolute-complexity
+    # gating of new models is a separate (future) feature.
+    regressions = [
+        d.unique_id
+        for d in deltas
+        if not d.is_new and d.unique_id not in waivers and d.delta > threshold
+    ]
+    passed = config.gate.mode != "fail" or not regressions
+    return GateReport(deltas=deltas, regressions=regressions, passed=passed, mode=config.gate.mode)

sqlquality/keys.py

@@ -0,0 +1,106 @@
+"""Redshift DISTKEY/SORTKEY inference from a model's JOIN/FILTER columns."""
+
+from __future__ import annotations
+
+from collections import Counter
+
+from sqlglot import exp
+
+from sqlquality.models import Finding, Severity
+from sqlquality.sqlast import SqlParseError, parse
+
+
+def _join_key_counts(tree: exp.Expression) -> Counter[str]:
+    """Count each column's occurrences across all equi-join predicates."""
+    counts: Counter[str] = Counter()
+    for join in tree.find_all(exp.Join):
+        on = join.args.get("on")
+        if on is None:
+            continue
+        for eq in on.find_all(exp.EQ):
+            left, right = eq.this, eq.expression
+            if isinstance(left, exp.Column) and isinstance(right, exp.Column):
+                # Dedupe per predicate so `a.k = b.k` counts `k` once, not twice, and
+                # cannot outweigh a column that appears across two separate joins.
+                counts.update({left.name, right.name})
+    return counts
+
+
+def _join_key_columns(tree: exp.Expression) -> list[str]:
+    return sorted(_join_key_counts(tree))
+
+
+def _filter_columns(tree: exp.Expression) -> list[str]:
+    where = tree.find(exp.Where)
+    if where is None:
+        return []
+    names: set[str] = set()
+    for node in where.find_all(exp.EQ, exp.GT, exp.LT, exp.GTE, exp.LTE, exp.Between, exp.In):
+        if isinstance(node, exp.Between):
+            col = node.this
+            low = node.args.get("low")
+            high = node.args.get("high")
+            if (
+                isinstance(col, exp.Column)
+                and isinstance(low, exp.Literal)
+                and isinstance(high, exp.Literal)
+            ):
+                names.add(col.name)
+        elif isinstance(node, exp.In):
+            col = node.this
+            values = node.args.get("expressions") or []
+            if (
+                isinstance(col, exp.Column)
+                and values
+                and all(isinstance(value, exp.Literal) for value in values)
+            ):
+                names.add(col.name)
+        else:
+            left, right = node.this, node.expression
+            if isinstance(left, exp.Column) and isinstance(right, exp.Literal):
+                names.add(left.name)
+            elif isinstance(right, exp.Column) and isinstance(left, exp.Literal):
+                names.add(right.name)
+    return sorted(names)
+
+
+def join_key_columns(sql: str, dialect: str) -> list[str]:
+    return _join_key_columns(parse(sql, dialect))
+
+
+def filter_columns(sql: str, dialect: str) -> list[str]:
+    return _filter_columns(parse(sql, dialect))
+
+
+def dist_sort_findings(sql: str, dialect: str) -> list[Finding]:
+    """Suggest a DISTKEY (join keys) and SORTKEY (filter columns) for the model."""
+    try:
+        tree = parse(sql, dialect)
+    except SqlParseError:
+        return []
+    findings: list[Finding] = []
+    counts = _join_key_counts(tree)
+    if counts:
+        # Redshift allows only ONE DISTKEY column: pick the most frequent equi-join
+        # key (ties broken alphabetically); surface the rest as alternates.
+        best = min(counts, key=lambda name: (-counts[name], name))
+        alternates = sorted(name for name in counts if name != best)
+        message = (
+            f"Consider a single-column DISTKEY on the most frequent join key: {best} "
+            "— colocates joined rows and avoids redistribution (Redshift permits only one DISTKEY)."
+        )
+        if alternates:
+            message += f" Alternate candidate(s): {', '.join(alternates)}."
+        findings.append(Finding("RS001", message, 0, Severity.INFO, False))
+    fc = _filter_columns(tree)
+    if fc:
+        findings.append(
+            Finding(
+                "RS002",
+                f"Consider a compound SORTKEY leading with the filter column(s): {', '.join(fc)} — enables zone-map block skipping.",
+                0,
+                Severity.INFO,
+                False,
+            )
+        )
+    return findings

sqlquality/linter.py

@@ -0,0 +1,70 @@
+"""Best-practice linting + auto-fix via SQLFluff's programmatic API."""
+
+from __future__ import annotations
+
+import sqlfluff
+
+from sqlquality.models import Finding, Severity
+
+# Templating-caused codes: raised when SQLFluff's Jinja templater can't resolve a
+# macro (dbt_utils.*, custom macros). TMP is the undefined-variable error; PRS is
+# the unparsable section that follows. On a raw dbt model these are advisory only.
+_TEMPLATING_CODES = frozenset({"TMP", "PRS"})
+_JINJA_HINT = (
+    " (unresolved Jinja — lint the compiled SQL under target/compiled/ "
+    "or pass a dbt-templater config via --sqlfluff-config)"
+)
+
+
+def _to_finding(violation: dict, *, templating: bool) -> Finding:
+    code = violation.get("code", "")
+    message = violation.get("description", "")
+    if templating and code in _TEMPLATING_CODES:
+        # Unresolvable-Jinja noise the user can't fix on the raw model: advise, never gate.
+        severity = Severity.INFO
+        message = message + _JINJA_HINT
+    elif code == "PRS":
+        # Genuine parse error on plain SQL.
+        severity = Severity.ERROR
+    else:
+        severity = Severity.WARNING
+    # PRS (parse-error) dicts have no "fixes" key — always use .get().
+    fixes = violation.get("fixes") or []
+    return Finding(
+        code=code,
+        message=message,
+        line=violation.get("start_line_no", 0),
+        severity=severity,
+        fixable=bool(fixes),
+    )
+
+
+def lint_sql(
+    sql: str,
+    dialect: str,
+    exclude_rules: list[str] | None = None,
+    config_path: str | None = None,
+) -> list[Finding]:
+    """Lint one SQL string; return findings (parse errors included as PRS).
+
+    A file whose violations include a TMP code is treated as unresolved-templating:
+    its TMP/PRS findings are demoted to INFO. TMP is the only reliable evidence that
+    templating failed — a genuine PRS on rendered SQL (Jinja only in comments, valid
+    Jinja that renders to broken SQL, templater=raw configs) carries no TMP and stays
+    ERROR.
+    """
+    violations = sqlfluff.lint(
+        sql, dialect=dialect, exclude_rules=exclude_rules, config_path=config_path
+    )
+    templating = any(v.get("code") == "TMP" for v in violations)
+    return [_to_finding(v, templating=templating) for v in violations]
+
+
+def fix_sql(
+    sql: str,
+    dialect: str,
+    exclude_rules: list[str] | None = None,
+    config_path: str | None = None,
+) -> str:
+    """Return SQL with SQLFluff auto-fixes applied (unchanged if unparseable)."""
+    return sqlfluff.fix(sql, dialect=dialect, exclude_rules=exclude_rules, config_path=config_path)

sqlquality/llm.py

@@ -0,0 +1,101 @@
+"""Optional, provider-agnostic LLM layer for enriching findings (advisory only)."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any, Protocol, runtime_checkable
+
+from sqlquality.models import Finding
+
+MAX_PROMPT_SQL_CHARS = 20_000
+
+
+@runtime_checkable
+class LLMProvider(Protocol):
+    """Anything that can turn a prompt into a suggestion string."""
+
+    def suggest(self, prompt: str) -> str: ...
+
+
+@dataclass(frozen=True)
+class Suggestion:
+    code: str
+    text: str
+
+
+def build_prompt(finding: Finding, sql: str) -> str:
+    """Build the prompt asking for a concrete fix for one finding."""
+    if len(sql) > MAX_PROMPT_SQL_CHARS:
+        sql = sql[:MAX_PROMPT_SQL_CHARS] + "... [truncated]"
+    return (
+        "You are a SQL performance and maintainability expert. "
+        "A static analyzer flagged this finding on a dbt model's SQL:\n"
+        f"- code: {finding.code}\n"
+        f"- message: {finding.message}\n\n"
+        f"SQL:\n{sql}\n\n"
+        "Suggest a concrete, minimal rewrite or configuration change that "
+        "addresses the finding. Be brief (2-4 sentences). If no change is "
+        "warranted, say so and explain why."
+    )
+
+
+class CallableProvider:
+    """Adapt any Callable[[str], str] into an LLMProvider (bring your own model)."""
+
+    def __init__(self, fn: Callable[[str], str]) -> None:
+        self._fn = fn
+
+    def suggest(self, prompt: str) -> str:
+        return self._fn(prompt)
+
+
+def enrich_findings(findings: list[Finding], sql: str, provider: LLMProvider) -> list[Suggestion]:
+    """One suggestion per finding. Advisory — never affects severity or the gate.
+
+    Each provider call is isolated: a failure on one finding is skipped rather
+    than aborting the rest, so a single bad call never discards every suggestion.
+    """
+    suggestions: list[Suggestion] = []
+    for f in findings:
+        try:
+            text = provider.suggest(build_prompt(f, sql))
+        except Exception:  # advisory-only: skip this finding, keep the others
+            continue
+        suggestions.append(Suggestion(f.code, text))
+    return suggestions
+
+
+class AnthropicProvider:
+    """LLM provider backed by the Anthropic Messages API (optional extra)."""
+
+    def __init__(self, model: str | None = None, client: object | None = None) -> None:
+        if client is None:
+            try:
+                import anthropic
+            except ImportError as exc:  # pragma: no cover - exercised only without the extra
+                raise RuntimeError(
+                    "The 'anthropic' package is required for AnthropicProvider. "
+                    "Install it with: pip install 'sqlquality[llm]'"
+                ) from exc
+            client = anthropic.Anthropic()
+        self._client: Any = client
+        self._model = model or os.environ.get("SQLQUALITY_LLM_MODEL", "claude-opus-4-8")
+
+    def suggest(self, prompt: str) -> str:
+        message = self._client.messages.create(
+            model=self._model,
+            max_tokens=1024,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        return "".join(
+            block.text for block in message.content if getattr(block, "type", None) == "text"
+        )
+
+
+def resolve_provider() -> LLMProvider | None:
+    """Return a provider if configured via SQLQUALITY_LLM, else None (off by default)."""
+    if os.environ.get("SQLQUALITY_LLM", "").strip().lower() not in {"anthropic", "1", "true"}:
+        return None
+    return AnthropicProvider()

sqlquality/models.py

@@ -0,0 +1,56 @@
+"""Shared data models for sqlquality."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+@dataclass(frozen=True)
+class ComplexityMetrics:
+    """Raw structural complexity counts for one SQL statement."""
+
+    join_count: int
+    cte_count: int
+    subquery_count: int
+    window_count: int
+    case_count: int
+    union_count: int
+    distinct_count: int
+    select_count: int
+    max_select_depth: int
+    projected_columns: int
+
+
+@dataclass(frozen=True)
+class DagFacts:
+    """A model's position in the dbt DAG (0 when unknown/offline)."""
+
+    fan_in: int = 0
+    fan_out: int = 0
+    lineage_depth: int = 0
+
+
+@dataclass(frozen=True)
+class ComplexityScore:
+    """A weighted, open-ended complexity score with per-component contributions."""
+
+    composite: float
+    components: dict[str, float]
+    metrics: ComplexityMetrics
+    dag: DagFacts | None = None
+
+
+class Severity(str, Enum):
+    INFO = "info"
+    WARNING = "warning"
+    ERROR = "error"
+
+
+@dataclass(frozen=True)
+class Finding:
+    code: str
+    message: str
+    line: int
+    severity: Severity
+    fixable: bool