dblect 0.1.0__py3-none-any.whl

dblect/__init__.py

@@ -0,0 +1,28 @@
+"""dblect: semantic correctness framework for dbt analytics pipelines.
+
+The authored surface a project writes against: domain types and model contracts
+(also under ``dblect.types``), and the contract/proxy layer (``contract``,
+``models``) for relating columns across a model's rows and across models. More
+surfaces (flags, the CLI) land as they are built.
+"""
+
+from dblect._version import __version__
+from dblect.contracts import contract, models
+from dblect.types import (
+    DomainType,
+    Field,
+    ForeignKey,
+    ModelContract,
+    PrimaryKey,
+)
+
+__all__ = [
+    "DomainType",
+    "Field",
+    "ForeignKey",
+    "ModelContract",
+    "PrimaryKey",
+    "__version__",
+    "contract",
+    "models",
+]

dblect/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

dblect/adapters/__init__.py

@@ -0,0 +1,40 @@
+"""Target adapters: one :class:`AdapterProfile` per dbt adapter, behind a registry.
+
+A dbt project compiles against one adapter (duckdb, snowflake, bigquery, ...), and
+that single choice fixes everything dblect reasons about the target: which sqlglot
+dialect parses its compiled SQL, whether the warehouse enforces PRIMARY KEY /
+UNIQUE and NOT NULL on write, and which incremental strategy runs when a model
+leaves ``incremental_strategy`` unset. :class:`AdapterProfile` gathers those facets
+into one value so a run reads a single coherent target.
+
+Adding a warehouse is self-contained: a module that builds an ``AdapterProfile``
+and calls :func:`register`. The built-ins under :mod:`dblect.adapters.builtin`
+are auto-discovered, so this package never enumerates them.
+
+An adapter is **validated** when dblect's detectors have been exercised against its
+SQL end-to-end (today, only duckdb). The others carry the runtime semantics dbt's
+adapter docs describe and route through the matching sqlglot dialect once the
+operator opts in via ``--dialect``.
+"""
+
+from __future__ import annotations
+
+from dblect.adapters.model import DEDUP_STRATEGIES, AdapterProfile, IncrementalStrategy
+from dblect.adapters.registry import (
+    UnvalidatedAdapterError,
+    profile_for_adapter,
+    register,
+    resolve_profile,
+    validated_adapters,
+)
+
+__all__ = [
+    "DEDUP_STRATEGIES",
+    "AdapterProfile",
+    "IncrementalStrategy",
+    "UnvalidatedAdapterError",
+    "profile_for_adapter",
+    "register",
+    "resolve_profile",
+    "validated_adapters",
+]

dblect/adapters/builtin/__init__.py

@@ -0,0 +1,7 @@
+"""Built-in adapter profiles, one module per warehouse.
+
+Each module builds an :class:`~dblect.adapters.model.AdapterProfile` and calls
+:func:`~dblect.adapters.registry.register`. The registry discovers every module in
+this package automatically, so adding a warehouse is a new file here and nothing
+else.
+"""

dblect/adapters/builtin/bigquery.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from dblect.adapters import AdapterProfile, IncrementalStrategy, register
+from dblect.sql import PORTABLE_NON_DETERMINISTIC_BUILTINS
+
+register(
+    AdapterProfile(
+        adapter_type="bigquery",
+        sqlglot_dialect="bigquery",
+        validated=True,
+        # NOT NULL is enforced (REQUIRED mode). PRIMARY KEY / FOREIGN KEY constraints
+        # exist but are advisory (unenforced), so keys are not enforced.
+        not_null_enforced=True,
+        key_enforced=False,
+        default_incremental_strategy=IncrementalStrategy.MERGE,
+        non_deterministic_builtins=PORTABLE_NON_DETERMINISTIC_BUILTINS,
+    )
+)

dblect/adapters/builtin/duckdb.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from dblect.adapters import AdapterProfile, register
+from dblect.sql import PORTABLE_NON_DETERMINISTIC_BUILTINS
+
+# `txid_current()` and `nextval()` arrive as `exp.Anonymous`. `random()`, `uuid()`,
+# `today()` and `gen_random_uuid()` are usually normalised to a typed node (so
+# already caught dialect-neutrally) and are listed for robustness, so a sqlglot
+# version that leaves them anonymous still fires.
+_DUCKDB_NON_DETERMINISTIC_BUILTINS = PORTABLE_NON_DETERMINISTIC_BUILTINS | frozenset(
+    {"random", "uuid", "txid_current", "nextval", "today", "gen_random_uuid"}
+)
+
+register(
+    AdapterProfile(
+        adapter_type="duckdb",
+        sqlglot_dialect="duckdb",
+        validated=True,
+        not_null_enforced=True,
+        key_enforced=True,
+        default_incremental_strategy=None,  # left unset pending validation
+        non_deterministic_builtins=_DUCKDB_NON_DETERMINISTIC_BUILTINS,
+    )
+)

dblect/adapters/builtin/postgres.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from dblect.adapters import AdapterProfile, IncrementalStrategy, register
+from dblect.sql import PORTABLE_NON_DETERMINISTIC_BUILTINS
+
+register(
+    AdapterProfile(
+        adapter_type="postgres",
+        sqlglot_dialect="postgres",
+        validated=False,
+        not_null_enforced=True,
+        key_enforced=True,
+        # dbt-postgres defaults to delete+insert (not merge) once a unique_key is set
+        default_incremental_strategy=IncrementalStrategy.DELETE_INSERT,
+        non_deterministic_builtins=PORTABLE_NON_DETERMINISTIC_BUILTINS,
+    )
+)

dblect/adapters/builtin/redshift.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from dblect.adapters import AdapterProfile, IncrementalStrategy, register
+from dblect.sql import PORTABLE_NON_DETERMINISTIC_BUILTINS
+
+register(
+    AdapterProfile(
+        adapter_type="redshift",
+        sqlglot_dialect="redshift",
+        validated=False,
+        not_null_enforced=True,
+        key_enforced=False,
+        # dbt-redshift, like Postgres, defaults to delete+insert (not merge) with a unique_key
+        default_incremental_strategy=IncrementalStrategy.DELETE_INSERT,
+        non_deterministic_builtins=PORTABLE_NON_DETERMINISTIC_BUILTINS,
+    )
+)

dblect/adapters/builtin/snowflake.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from dblect.adapters import AdapterProfile, IncrementalStrategy, register
+from dblect.sql import PORTABLE_NON_DETERMINISTIC_BUILTINS
+
+register(
+    AdapterProfile(
+        adapter_type="snowflake",
+        sqlglot_dialect="snowflake",
+        validated=False,
+        not_null_enforced=True,
+        key_enforced=False,
+        default_incremental_strategy=IncrementalStrategy.MERGE,
+        non_deterministic_builtins=PORTABLE_NON_DETERMINISTIC_BUILTINS,
+    )
+)

dblect/adapters/model.py

@@ -0,0 +1,87 @@
+"""The value types describing one target adapter: :class:`AdapterProfile` and the
+incremental strategies it can default to.
+
+These carry no per-adapter data themselves; the concrete profiles live in
+self-contained modules under :mod:`dblect.adapters.builtin` and register
+themselves with the registry, so adding a warehouse never edits this file.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+
+
+class IncrementalStrategy(StrEnum):
+    """The incremental materialization strategies dbt ships.
+
+    A custom strategy (a project-defined macro) is outside this closed set;
+    :meth:`parse` maps an unrecognized value to ``None`` so a caller reads it as
+    "no known dedup guarantee" rather than guessing.
+    """
+
+    APPEND = "append"
+    MERGE = "merge"
+    DELETE_INSERT = "delete+insert"
+    INSERT_OVERWRITE = "insert_overwrite"
+    MICROBATCH = "microbatch"
+
+    @classmethod
+    def parse(cls, raw: str | None) -> IncrementalStrategy | None:
+        """The strategy a config string names, or ``None`` when unset or custom."""
+        if raw is None:
+            return None
+        try:
+            return cls(raw.strip().lower())
+        except ValueError:
+            return None
+
+
+# Strategies whose write deduplicates on ``unique_key``: ``merge`` updates the
+# matching row and ``delete+insert`` removes then reinserts, so each key value
+# lands once. ``append`` inserts unconditionally and ``insert_overwrite`` replaces
+# whole partitions, so neither enforces the key.
+DEDUP_STRATEGIES: frozenset[IncrementalStrategy] = frozenset(
+    {IncrementalStrategy.MERGE, IncrementalStrategy.DELETE_INSERT}
+)
+
+
+@dataclass(frozen=True, slots=True)
+class AdapterProfile:
+    """Everything dblect's analysis needs to know about one target adapter.
+
+    ``adapter_type`` is the effective target's dbt name (the manifest's, or the one
+    an override selects); ``sqlglot_dialect`` parses its compiled SQL. The two
+    enforcement flags are descriptive provenance, read by the unenforced-constraint
+    findings and never by fact resolution. ``default_incremental_strategy`` is the
+    strategy in force when a model leaves ``incremental_strategy`` unset, or
+    ``None`` where dblect does not know the adapter's default to deduplicate.
+    ``non_deterministic_builtins`` is the complete name set (the portable baseline
+    plus this adapter's own builtins) the non-determinism detector matches anonymous
+    function calls against; a consumer reads it whole and unions nothing.
+    ``duplicate_safe_aggregate_builtins`` is the same shape for the duplicate-sensitivity
+    predicate: names of UDF aggregates this warehouse exposes (parsed as anonymous
+    calls) whose result a duplicated row does not change, so a fan-out into them is
+    harmless. It defaults to empty, since most duplicate-safe aggregates already carry a
+    dedicated sqlglot type; the few that arrive anonymous are declared here.
+    """
+
+    adapter_type: str
+    sqlglot_dialect: str
+    validated: bool
+    not_null_enforced: bool
+    key_enforced: bool
+    default_incremental_strategy: IncrementalStrategy | None
+    non_deterministic_builtins: frozenset[str]
+    duplicate_safe_aggregate_builtins: frozenset[str] = frozenset()
+
+    def effective_strategy(self, declared: str | None) -> IncrementalStrategy | None:
+        """The strategy in force for a model: the declared one when set (``None`` if
+        that is a custom strategy dblect does not recognize, so no default is
+        assumed for a model that did choose a strategy), else this adapter's
+        default. The default branch is meaningful only when a ``unique_key`` is
+        present, which the config discoverer checks before consulting it.
+        """
+        if declared is not None:
+            return IncrementalStrategy.parse(declared)
+        return self.default_incremental_strategy

dblect/adapters/registry.py

@@ -0,0 +1,108 @@
+"""The adapter registry: registration, auto-discovery, and resolution.
+
+Built-in profiles live one per module under :mod:`dblect.adapters.builtin` and are
+auto-discovered on first lookup, so nothing here enumerates them. See the package
+docstring for how the pieces fit together.
+"""
+
+from __future__ import annotations
+
+import importlib
+import pkgutil
+
+from dblect.adapters.model import AdapterProfile
+from dblect.sql import PORTABLE_NON_DETERMINISTIC_BUILTINS
+
+_PROFILES: dict[str, AdapterProfile] = {}
+_loaded = False
+
+
+def register(profile: AdapterProfile) -> AdapterProfile:
+    """Register ``profile`` under its (case-folded) adapter name and return it, so a
+    module can both register and keep a reference: ``SNOWFLAKE = register(...)``.
+
+    A later registration of the same name wins, which lets a host refine or replace
+    a built-in profile without editing it.
+    """
+    _PROFILES[profile.adapter_type.strip().lower()] = profile
+    return profile
+
+
+def _ensure_loaded() -> None:
+    """Import the built-in adapter modules once, so their ``register`` calls run.
+
+    Discovery walks the :mod:`dblect.adapters.builtin` package rather than naming
+    its modules, so a new built-in is picked up without being listed here. The flag
+    is set before importing so a module that triggers a lookup during its own
+    import does not recurse.
+    """
+    global _loaded
+    if _loaded:
+        return
+    _loaded = True
+    from dblect.adapters import builtin
+
+    for info in pkgutil.iter_modules(builtin.__path__, builtin.__name__ + "."):
+        importlib.import_module(info.name)
+
+
+def _conservative(adapter_type: str, *, sqlglot_dialect: str | None = None) -> AdapterProfile:
+    """The profile for an adapter dblect has no specific knowledge of: NOT NULL
+    enforced (true on essentially every warehouse), PRIMARY KEY / UNIQUE advisory,
+    no known dedup default (so an unset incremental strategy claims no key), and
+    only the portable non-determinism baseline."""
+    return AdapterProfile(
+        adapter_type=adapter_type,
+        sqlglot_dialect=sqlglot_dialect if sqlglot_dialect is not None else adapter_type,
+        validated=False,
+        not_null_enforced=True,
+        key_enforced=False,
+        default_incremental_strategy=None,
+        non_deterministic_builtins=PORTABLE_NON_DETERMINISTIC_BUILTINS,
+    )
+
+
+def profile_for_adapter(adapter_type: str) -> AdapterProfile:
+    """The capability profile for a dbt adapter by name.
+
+    An adapter no module has registered gets a conservative profile, never an
+    error: this is the semantics lookup, distinct from the parsing-validation gate
+    in :func:`resolve_profile`.
+    """
+    _ensure_loaded()
+    return _PROFILES.get(adapter_type.strip().lower()) or _conservative(adapter_type)
+
+
+def validated_adapters() -> frozenset[str]:
+    """The names of registered adapters dblect has validated end-to-end."""
+    _ensure_loaded()
+    return frozenset(name for name, profile in _PROFILES.items() if profile.validated)
+
+
+class UnvalidatedAdapterError(ValueError):
+    """The manifest's adapter is not in dblect's validated set and no ``--dialect``
+    override is in effect. Carries the adapter name so the CLI can build an
+    actionable message."""
+
+    def __init__(self, adapter_type: str) -> None:
+        super().__init__(
+            f"adapter `{adapter_type}` is not in dblect's validated set "
+            f"({sorted(validated_adapters())})"
+        )
+        self.adapter_type = adapter_type
+
+
+def resolve_profile(*, adapter_type: str, explicit_dialect: str | None) -> AdapterProfile:
+    """The single target profile for a run, or raise :class:`UnvalidatedAdapterError`.
+
+    An ``explicit_dialect`` override names the target wholesale (its grammar and its
+    runtime semantics together), so the two cannot drift apart; passing it is the
+    operator's acknowledgment of a best-effort, possibly unvalidated
+    interpretation. Without an override the manifest's adapter must be validated.
+    """
+    if explicit_dialect is not None:
+        return profile_for_adapter(explicit_dialect)
+    profile = profile_for_adapter(adapter_type)
+    if not profile.validated:
+        raise UnvalidatedAdapterError(adapter_type)
+    return profile

dblect/analysis.py

@@ -0,0 +1,125 @@
+"""One analysis door: run every detector family over a manifest, return every
+finding under one sealed type.
+
+Two families surface findings today. :func:`~dblect.check.run.run_check` reports
+the declaration-level ones (contract resolution, domain-type contradictions across
+the DAG, not-well-typed aggregations), located by model, column, and contract.
+:func:`~dblect.audit.run_audit` reports the SQL-structural ones (join fan-out,
+window order, the nullability hazards, the rest), located by a span in one compiled
+statement. The two stay distinct in representation and altitude; issue #107 weighs
+whether to merge the representations as well.
+
+What this module removes is the obligation to *know* there are two families. Asking
+for "the findings of a manifest" should not require remembering to call both
+producers and merge them: a consumer that threads findings (the incremental-worlds
+cross-world diff, and the world axes still to come) calls :func:`analyze` once and
+gets them all. Adding a third family is a change here, not in every consumer, and
+:data:`AnalysisFinding` is sealed so a ``match`` over it with ``assert_never`` turns
+a forgotten family into a type error rather than a silent coverage gap. That gap is
+not hypothetical: the incremental check first threaded only the declaration-level
+family, so the structural detectors (the very hazard it exists to catch) were absent
+until the omission was noticed by hand.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Hashable
+from dataclasses import dataclass
+from typing import assert_never
+
+from dblect.adapters import AdapterProfile
+from dblect.audit import AuditReport, LocatedFinding, run_audit
+from dblect.check.findings import CheckFinding, CheckReport
+from dblect.check.run import build_check_graphs, run_check
+from dblect.manifest import Manifest
+from dblect.sql import parse_manifest_models
+from dblect.types import ContractRegistry
+
+# The sealed set of findings any analysis surfaces: one member per detector family.
+# A ``match`` over this union closed by ``assert_never`` is exhaustiveness-checked,
+# so adding a family without handling it everywhere is a type error, not a quiet
+# blind spot.
+AnalysisFinding = CheckFinding | LocatedFinding
+
+# A finding's identity across two compilations of the same project: enough to say
+# "the same issue in both worlds" while ignoring what drifts between compiled SQLs.
+FindingIdentity = tuple[Hashable, ...]
+
+
+def cross_world_identity(finding: AnalysisFinding) -> FindingIdentity:
+    """The stable cross-world identity of ``finding``: where it lands and what it is,
+    without the message or line span that differ between two compilations. A
+    declaration-level finding keys on kind/model/column/contract; a structural one on
+    kind/model and the rendered offending snippet. A snippet present in one world only
+    (a steady-state-only join) has no match in the other, so it surfaces as varying.
+    """
+    match finding:
+        case CheckFinding():
+            return (
+                "check",
+                finding.kind,
+                finding.model_unique_id,
+                finding.column,
+                finding.contract,
+            )
+        case LocatedFinding():
+            inner = finding.finding
+            return ("audit", inner.kind, finding.model_unique_id, inner.sql_snippet)
+    assert_never(finding)
+
+
+@dataclass(frozen=True, slots=True)
+class AnalysisReport:
+    """Every finding for one manifest, from every detector family, plus each family's
+    own report so a caller that needs the family-specific extras (coverage blocks,
+    suppressed directives) still has them. ``findings`` is the merged, sealed view the
+    finding-threading consumers read; the rest is there when an altitude matters."""
+
+    findings: tuple[AnalysisFinding, ...]
+    check: CheckReport
+    audit: AuditReport
+
+
+def analyze(
+    manifest: Manifest,
+    profile: AdapterProfile,
+    *,
+    registry: ContractRegistry | None = None,
+    resolution_floor: float | None = None,
+) -> AnalysisReport:
+    """Run every detector family over ``manifest`` and return every finding.
+
+    ``profile`` is the resolved target whose dialect parses every model, and
+    ``registry`` the contracts to resolve (defaulting to the active one), the same
+    inputs :func:`~dblect.check.run.run_check` takes. ``resolution_floor`` is forwarded
+    to it unchanged. The merged ``findings`` carry both families so a consumer never
+    has to enumerate the families itself.
+
+    The two families share one parse and one lineage-graph build (the run's dominant cost),
+    threaded into both. The shared build stamps the parsed trees in place and ``propagate``
+    never mutates them, so both families read the same resolution they would have built
+    themselves. ``run_check`` takes ``graphs`` rather than ``registry`` here, since ``graphs``
+    already carries the resolved contracts.
+
+    The build's resolved ``determines`` facts are also threaded into the structural audit so
+    join-fanout grounds key coverage through functional dependencies (a declared ``wiki_id
+    determines wiki_name`` lets a join on the determinant cover a key carrying the dependent).
+    The declaration family already reads these off the shared build; this hands the same facts
+    to the structural one.
+    """
+    parsed, trees = parse_manifest_models(manifest, dialect=profile.sqlglot_dialect)
+    graphs = build_check_graphs(manifest, profile, registry=registry, trees=trees)
+    check = run_check(manifest, profile, resolution_floor=resolution_floor, graphs=graphs)
+    audit = run_audit(
+        manifest,
+        profile,
+        parsed=parsed,
+        column_graph=graphs.column_build.graph,
+        relation_graph=graphs.relation_build.graph,
+        fd_facts=graphs.resolved.fd_facts,
+    )
+    return AnalysisReport(
+        findings=(*check.findings, *audit.findings),
+        check=check,
+        audit=audit,
+    )

dblect/audit/__init__.py

@@ -0,0 +1,26 @@
+"""Audit pipeline: static detectors, replay-determinism, heuristic invariants."""
+
+from dblect.audit.sourcemap import SourceSpan, SpanBasis
+from dblect.audit.suppress import SuppressionDirective
+from dblect.audit.walker import (
+    DEFAULT_DETECTORS,
+    AuditReport,
+    Detector,
+    LocatedFinding,
+    SkippedModel,
+    SuppressedFinding,
+    run_audit,
+)
+
+__all__ = [
+    "DEFAULT_DETECTORS",
+    "AuditReport",
+    "Detector",
+    "LocatedFinding",
+    "SkippedModel",
+    "SourceSpan",
+    "SpanBasis",
+    "SuppressedFinding",
+    "SuppressionDirective",
+    "run_audit",
+]