qualipilot 2.0.0__py3-none-any.whl

qualipilot/__init__.py

@@ -0,0 +1,16 @@
+"""Production-grade data quality checker with pluggable LLM backends."""
+
+from qualipilot.checker import DataQualityChecker
+from qualipilot.models.config import CheckConfig, LLMConfig, QualipilotConfig
+from qualipilot.models.results import CheckResult, QualityReport
+
+__all__ = [
+    "CheckConfig",
+    "CheckResult",
+    "DataQualityChecker",
+    "LLMConfig",
+    "QualipilotConfig",
+    "QualityReport",
+]
+
+__version__ = "2.0.0"

qualipilot/checker.py

@@ -0,0 +1,195 @@
+"""Orchestrator tying engines, checks and LLM reporting together."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from qualipilot.checks import (
+    CardinalityCheck,
+    Check,
+    CheckContext,
+    DataTypesCheck,
+    DuplicatesCheck,
+    FreshnessCheck,
+    LinkageCheck,
+    MissingValuesCheck,
+    OutliersCheck,
+    RangesCheck,
+)
+from qualipilot.engines import build_engine
+from qualipilot.models.config import QualipilotConfig
+from qualipilot.models.results import (
+    CheckResult,
+    DatasetStats,
+    QualityReport,
+)
+
+if TYPE_CHECKING:
+    from qualipilot.engines.base import Engine
+    from qualipilot.llm.base import LLMProvider
+
+logger = logging.getLogger(__name__)
+
+
+class DataQualityChecker:
+    """Run configurable data quality checks against a dataframe/file.
+
+    Example:
+        >>> import pandas as pd
+        >>> from qualipilot import DataQualityChecker, QualipilotConfig
+        >>> df = pd.read_csv("orders.csv")
+        >>> checker = DataQualityChecker(df, QualipilotConfig())
+        >>> report = checker.run()
+        >>> print(report.to_json())
+    """
+
+    def __init__(
+        self,
+        data: Any,
+        config: QualipilotConfig | None = None,
+    ) -> None:
+        self.config = config or QualipilotConfig()
+        self.engine: Engine = build_engine(data, kind=self.config.engine)
+        logger.info(
+            "initialised checker with %s engine over %d rows",
+            self.engine.name,
+            self.engine.row_count(),
+        )
+
+    def run(self) -> QualityReport:
+        """Run every enabled check and return the aggregate report."""
+        ctx = CheckContext(engine=self.engine, config=self.config.checks)
+        results: list[CheckResult] = []
+        for check in self._build_check_list():
+            logger.info("running check: %s", check.name)
+            result = check.run(ctx)
+            logger.info(
+                "check %s finished in %.3fs (severity=%s)",
+                result.name,
+                result.duration_seconds,
+                result.severity,
+            )
+            results.append(result)
+
+        dataset = DatasetStats(
+            row_count=self.engine.row_count(),
+            column_count=len(self.engine.columns()),
+            columns=self.engine.columns(),
+            dtypes=self.engine.dtypes(),
+            engine=self.engine.name,
+        )
+
+        report = QualityReport(
+            dataset=dataset,
+            results=results,
+            config_hash=_config_fingerprint(self.config),
+        )
+
+        llm_report = self._maybe_render_llm_report(report)
+        if llm_report:
+            report.llm_report = llm_report
+
+        if self.config.output_path is not None:
+            self.save(report, self.config.output_path)
+
+        return report
+
+    # ---- helpers ------------------------------------------------------
+
+    def _build_check_list(self) -> list[Check]:
+        cfg = self.config.checks
+        checks: list[Check] = []
+        if cfg.missing_values:
+            checks.append(MissingValuesCheck())
+        if cfg.duplicates:
+            checks.append(DuplicatesCheck())
+        if cfg.data_types:
+            checks.append(DataTypesCheck())
+        if cfg.outliers:
+            checks.append(OutliersCheck())
+        if cfg.ranges:
+            checks.append(RangesCheck())
+        if cfg.cardinality:
+            checks.append(CardinalityCheck())
+        if cfg.freshness:
+            checks.append(FreshnessCheck())
+        if cfg.linkage is not None:
+            checks.append(LinkageCheck())
+        return checks
+
+    def _maybe_render_llm_report(self, report: QualityReport) -> str | None:
+        provider_name = self.config.llm.provider
+        if provider_name == "none":
+            return None
+        provider = _build_llm_provider(self.config.llm)
+        prompt = _build_llm_prompt(report)
+        try:
+            return provider.generate(
+                system=self.config.llm.system_prompt, user=prompt
+            )
+        except Exception as exc:
+            # llm problems should not fail the check pipeline
+            logger.error("llm report generation failed: %s", exc)
+            return f"LLM report failed: {exc}"
+
+    @staticmethod
+    def save(report: QualityReport, path: str | Path) -> None:
+        """Persist the JSON report to disk (utf-8, pretty-printed)."""
+        target = Path(path)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(report.to_json(), encoding="utf-8")
+        logger.info("report written to %s", target)
+
+
+def _config_fingerprint(cfg: QualipilotConfig) -> str:
+    """Stable SHA-1 of the config, handy for report dedup in pipelines."""
+    payload = cfg.model_dump_json(exclude={"output_path"})
+    return hashlib.sha1(
+        payload.encode("utf-8"), usedforsecurity=False
+    ).hexdigest()
+
+
+def _build_llm_provider(cfg: Any) -> LLMProvider:
+    # late import so the optional boto3/openai deps stay optional
+    from qualipilot.llm import build_provider
+
+    return build_provider(cfg)
+
+
+def _build_llm_prompt(report: QualityReport) -> str:
+    # compact the report before shipping to the model so we do not
+    # waste tokens on huge sample arrays
+    compact = {
+        "dataset": report.dataset.model_dump(),
+        "results": [
+            {
+                "name": r.name,
+                "severity": r.severity,
+                "duration_seconds": round(r.duration_seconds, 3),
+                "summary": _summarise_payload(r.payload),
+                "error": r.error,
+            }
+            for r in report.results
+        ],
+    }
+    return (
+        "Analyse this data quality report and produce actionable "
+        "findings. Keep samples out; focus on what to fix.\n\n"
+        + json.dumps(compact, default=str)
+    )
+
+
+def _summarise_payload(payload: dict[str, Any]) -> dict[str, Any]:
+    """Strip large sample arrays so prompts stay cheap."""
+    keep: dict[str, Any] = {}
+    for key, value in payload.items():
+        if isinstance(value, list):
+            # keep length only, drop row-level samples
+            keep[key] = {"count": len(value)}
+        else:
+            keep[key] = value
+    return keep

qualipilot/checks/__init__.py

@@ -0,0 +1,28 @@
+"""Individual data quality checks.
+
+Each module implements a single ``Check`` subclass. The orchestrator
+in ``qualipilot.checker`` picks which to run based on ``CheckConfig``.
+"""
+
+from qualipilot.checks.base import Check, CheckContext
+from qualipilot.checks.cardinality import CardinalityCheck
+from qualipilot.checks.duplicates import DuplicatesCheck
+from qualipilot.checks.freshness import FreshnessCheck
+from qualipilot.checks.linkage import LinkageCheck
+from qualipilot.checks.missing import MissingValuesCheck
+from qualipilot.checks.outliers import OutliersCheck
+from qualipilot.checks.ranges import RangesCheck
+from qualipilot.checks.types import DataTypesCheck
+
+__all__ = [
+    "CardinalityCheck",
+    "Check",
+    "CheckContext",
+    "DataTypesCheck",
+    "DuplicatesCheck",
+    "FreshnessCheck",
+    "LinkageCheck",
+    "MissingValuesCheck",
+    "OutliersCheck",
+    "RangesCheck",
+]

qualipilot/checks/base.py

@@ -0,0 +1,61 @@
+"""Base types shared by every check."""
+
+from __future__ import annotations
+
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any
+
+from qualipilot.engines.base import Engine
+from qualipilot.models.config import CheckConfig
+from qualipilot.models.results import CheckResult
+
+
+@dataclass(slots=True)
+class CheckContext:
+    """Immutable inputs handed to every check."""
+
+    engine: Engine
+    config: CheckConfig
+
+
+class Check(ABC):
+    """Base class for data quality checks.
+
+    Subclasses implement ``_execute`` which returns the serialisable
+    payload. The base class times execution, captures exceptions, and
+    assigns the severity.
+    """
+
+    name: str
+
+    def run(self, ctx: CheckContext) -> CheckResult:
+        """Execute the check, wrapping timing and error handling."""
+        start = time.perf_counter()
+        try:
+            severity, payload = self._execute(ctx)
+        except Exception as exc:
+            return CheckResult(
+                name=self.name,
+                severity="error",
+                duration_seconds=time.perf_counter() - start,
+                payload={},
+                error=f"{type(exc).__name__}: {exc}",
+            )
+        return CheckResult(
+            name=self.name,
+            severity=severity,
+            duration_seconds=time.perf_counter() - start,
+            payload=payload,
+        )
+
+    @abstractmethod
+    def _execute(self, ctx: CheckContext) -> tuple[str, dict[str, Any]]:
+        """Run the check logic.
+
+        Returns:
+            Tuple of ``(severity, payload)``. Severity is ``"ok"``,
+            ``"warn"`` or ``"error"``. Payload is a JSON-serialisable
+            dict stored on the resulting ``CheckResult``.
+        """

qualipilot/checks/cardinality.py

@@ -0,0 +1,42 @@
+"""Cardinality / uniqueness profile.
+
+Helps spot accidentally-constant columns and highly categorical ones
+that should be bucketed. Reports distinct count plus the top-10 values
+for every column.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from qualipilot.checks.base import Check, CheckContext
+
+
+class CardinalityCheck(Check):
+    name = "cardinality"
+
+    def _execute(self, ctx: CheckContext) -> tuple[str, dict[str, Any]]:
+        total_rows = ctx.engine.row_count() or 1
+        report: list[dict[str, Any]] = []
+        any_constant = False
+
+        for col in ctx.engine.columns():
+            try:
+                distinct = ctx.engine.distinct_count(col)
+            except Exception:  # pragma: no cover
+                # some engines fail on nested dtypes, we log + move on
+                continue
+            top = ctx.engine.top_values(col, n=10)
+            if distinct <= 1 and total_rows > 1:
+                any_constant = True
+            report.append(
+                {
+                    "column": col,
+                    "distinct_count": distinct,
+                    "unique_ratio": round(distinct / total_rows, 6),
+                    "top_values": top,
+                }
+            )
+
+        severity = "warn" if any_constant else "ok"
+        return severity, {"per_column": report}

qualipilot/checks/duplicates.py

@@ -0,0 +1,28 @@
+"""Duplicate row check.
+
+Uses a global duplicate count so distributed backends do not
+under-count the way ``map_partitions(.duplicated())`` would.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from qualipilot.checks.base import Check, CheckContext
+
+
+class DuplicatesCheck(Check):
+    name = "duplicates"
+
+    def _execute(self, ctx: CheckContext) -> tuple[str, dict[str, Any]]:
+        subset = ctx.config.duplicate_subset
+        total = ctx.engine.duplicate_count(subset=subset)
+        sample = ctx.engine.sample_duplicates(
+            n=ctx.config.sample_size, subset=subset
+        )
+        severity = "warn" if total > 0 else "ok"
+        return severity, {
+            "total_duplicate_rows": total,
+            "subset": subset,
+            "sample": sample,
+        }

qualipilot/checks/freshness.py

@@ -0,0 +1,77 @@
+"""Data freshness check.
+
+Flags datasets where the latest timestamp in configured columns is
+older than ``freshness_max_age_hours``. Useful for scheduled batch
+jobs that should never publish stale snapshots.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from qualipilot.checks.base import Check, CheckContext
+
+
+class FreshnessCheck(Check):
+    name = "freshness"
+
+    def _execute(self, ctx: CheckContext) -> tuple[str, dict[str, Any]]:
+        cols = ctx.config.freshness_columns or ctx.engine.datetime_columns()
+        if not cols:
+            return "ok", {"per_column": []}
+
+        max_age = timedelta(hours=ctx.config.freshness_max_age_hours)
+        now = datetime.now(UTC)
+        report: list[dict[str, Any]] = []
+        any_stale = False
+
+        for col in cols:
+            max_ts = ctx.engine.max_datetime(col)
+            if max_ts is None:
+                report.append(
+                    {
+                        "column": col,
+                        "max_timestamp": None,
+                        "is_stale": True,
+                        "note": "no non-null values",
+                    }
+                )
+                any_stale = True
+                continue
+
+            # normalise naive timestamps so subtraction is safe
+            ts = _as_aware(max_ts)
+            age = now - ts
+            stale = age > max_age
+            if stale:
+                any_stale = True
+            report.append(
+                {
+                    "column": col,
+                    "max_timestamp": ts.isoformat(),
+                    "age_hours": round(age.total_seconds() / 3600, 3),
+                    "is_stale": stale,
+                }
+            )
+
+        severity = "error" if any_stale else "ok"
+        return severity, {"per_column": report}
+
+
+def _as_aware(value: Any) -> datetime:
+    """Coerce engine-returned timestamps into timezone-aware datetime."""
+    # pandas.Timestamp, polars Datetime and stdlib datetime all satisfy
+    # the duck-typing we need; pandas ts has .to_pydatetime()
+    dt: datetime
+    if hasattr(value, "to_pydatetime"):
+        dt = value.to_pydatetime()
+    elif isinstance(value, datetime):
+        dt = value
+    else:
+        # last-resort coerce via iso parsing for exotic types
+        dt = datetime.fromisoformat(str(value))
+
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=UTC)
+    return dt

qualipilot/checks/linkage.py

@@ -0,0 +1,73 @@
+"""Optional probabilistic duplicate check.
+
+Unlike ``DuplicatesCheck`` (exact row match), this runs the
+Fellegi-Sunter linker on a user-supplied comparison spec and reports
+the number of probable duplicate clusters.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from qualipilot.checks.base import Check, CheckContext
+
+
+class LinkageCheck(Check):
+    """Report probabilistic duplicate clusters.
+
+    Active only when ``CheckConfig.linkage`` is populated.
+    """
+
+    name = "linkage"
+
+    def _execute(self, ctx: CheckContext) -> tuple[str, dict[str, Any]]:
+        link_cfg = ctx.config.linkage
+        if link_cfg is None:
+            return "ok", {"skipped": True}
+
+        # deferred import so the core install stays slim
+        from qualipilot.linking import RecordLinker
+
+        # engine-agnostic: always pull a polars frame for the linker.
+        # the per-engine cost is tiny compared with the linkage run.
+        pl_df = _engine_to_polars(ctx.engine)
+        result = RecordLinker(pl_df, link_cfg).run()
+        summary = result.summary()
+
+        # clusters of size > 1 are the "probable duplicate groups"
+        counts: dict[int, int] = {}
+        for cid in result.clusters.values():
+            counts[cid] = counts.get(cid, 0) + 1
+        multi = sum(1 for sz in counts.values() if sz > 1)
+        total_records_in_dupe_group = sum(
+            sz for sz in counts.values() if sz > 1
+        )
+
+        severity = "warn" if multi > 0 else "ok"
+        return severity, {
+            **summary,
+            "duplicate_clusters": multi,
+            "records_in_duplicate_groups": total_records_in_dupe_group,
+        }
+
+
+def _engine_to_polars(engine: Any) -> Any:
+    """Return the underlying frame as polars regardless of engine."""
+    # PolarsEngine stores a DataFrame directly
+    name = getattr(engine, "name", "")
+    if name == "polars":
+        return engine._df
+    if name == "pandas":
+        import polars as pl
+
+        return pl.from_pandas(engine._df)
+    # dask / cudf — round-trip via pandas
+    import polars as pl
+
+    underlying = engine._df
+    materialised = (
+        underlying.compute()
+        if hasattr(underlying, "compute")
+        else underlying.to_pandas()
+    )
+    return pl.from_pandas(materialised)

qualipilot/checks/missing.py

@@ -0,0 +1,43 @@
+"""Missing value check.
+
+Reports per-column null counts and percentages. Severity rules:
+    * any nulls -> warn
+    * columns where >50% are null -> error
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from qualipilot.checks.base import Check, CheckContext
+
+
+class MissingValuesCheck(Check):
+    name = "missing_values"
+
+    def _execute(self, ctx: CheckContext) -> tuple[str, dict[str, Any]]:
+        nulls = ctx.engine.null_counts()
+        total_rows = ctx.engine.row_count() or 1
+        stats = [
+            {
+                "column": col,
+                "null_count": int(count),
+                "null_percentage": round((count / total_rows) * 100, 4),
+            }
+            for col, count in nulls.items()
+        ]
+
+        total_nulls = sum(s["null_count"] for s in stats)
+        worst = max((s["null_percentage"] for s in stats), default=0.0)
+
+        severity = "ok"
+        if worst > 50:
+            severity = "error"
+        elif total_nulls > 0:
+            severity = "warn"
+
+        return severity, {
+            "total_null_count": total_nulls,
+            "worst_column_pct": worst,
+            "per_column": stats,
+        }

qualipilot/checks/outliers.py

@@ -0,0 +1,67 @@
+"""Outlier check using the IQR rule.
+
+Computes Q1 and Q3 for every numeric column in a single pass via the
+engine's batched ``quantiles`` API, then counts/samples values outside
+``[Q1 - k*IQR, Q3 + k*IQR]``.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+from qualipilot.checks.base import Check, CheckContext
+
+
+class OutliersCheck(Check):
+    name = "outliers"
+
+    def _execute(self, ctx: CheckContext) -> tuple[str, dict[str, Any]]:
+        numeric = ctx.engine.numeric_columns()
+        if not numeric:
+            return "ok", {"per_column": []}
+
+        qmap = ctx.engine.quantiles(numeric, qs=(0.25, 0.75))
+        k = ctx.config.outlier_iqr_multiplier
+
+        report: list[dict[str, Any]] = []
+        any_outliers = False
+        for col in numeric:
+            q1 = qmap[col][0.25]
+            q3 = qmap[col][0.75]
+            if _is_nan(q1) or _is_nan(q3):
+                # constant or empty column, nothing to flag
+                continue
+            iqr = q3 - q1
+            low = q1 - k * iqr
+            high = q3 + k * iqr
+            count = ctx.engine.count_outside(col, low, high)
+            sample = (
+                ctx.engine.sample_outside(
+                    col, low, high, ctx.config.sample_size
+                )
+                if count
+                else []
+            )
+            if count:
+                any_outliers = True
+            report.append(
+                {
+                    "column": col,
+                    "lower_bound": low,
+                    "upper_bound": high,
+                    "outlier_count": count,
+                    "sample": sample,
+                }
+            )
+
+        severity = "warn" if any_outliers else "ok"
+        return severity, {"per_column": report}
+
+
+def _is_nan(value: float) -> bool:
+    # protects against None-like sentinels alongside real nan floats
+    try:
+        return math.isnan(value)
+    except TypeError:
+        return False