framelint 0.1.0__py3-none-any.whl

framelint/__init__.py

@@ -0,0 +1,33 @@
+"""framelint: a lightweight data-quality profiler and CI gate for tabular data.
+
+Scan a :class:`pandas.DataFrame` or a CSV/Parquet file and get a clear,
+machine-readable data-quality report. The same report doubles as a CI gate:
+it can fail a build (non-zero exit code) when quality drops below configurable
+thresholds.
+
+Example:
+    >>> import framelint
+    >>> report = framelint.scan("sales.csv")
+    >>> report.summary()           # pretty console table
+    >>> report.passed              # bool, based on thresholds
+"""
+
+from __future__ import annotations
+
+from framelint.api import save_baseline, scan
+from framelint.config import ColumnRule, Config
+from framelint.models import Finding, Severity
+from framelint.report import Report
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "ColumnRule",
+    "Config",
+    "Finding",
+    "Report",
+    "Severity",
+    "__version__",
+    "save_baseline",
+    "scan",
+]

framelint/api.py

@@ -0,0 +1,73 @@
+"""The high-level public API: :func:`scan` and :func:`save_baseline`."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from framelint.baseline import BaselineLike, compute_drift
+from framelint.baseline import save_baseline as _save_baseline
+from framelint.checks import run_all_checks
+from framelint.config import Config
+from framelint.loaders import DataSource, load_data
+from framelint.report import Report
+
+
+def scan(
+    source: DataSource,
+    *,
+    config: Config | dict[str, Any] | None = None,
+    baseline: BaselineLike | None = None,
+) -> Report:
+    """Scan a dataset and return a data-quality :class:`Report`.
+
+    Args:
+        source: A :class:`pandas.DataFrame`, or a path to a CSV/Parquet file.
+        config: A :class:`Config`, a dict of overrides, or ``None`` for defaults.
+        baseline: An optional baseline (path or dict) to enable schema-drift
+            detection against a previously-captured schema.
+
+    Returns:
+        A :class:`Report` describing the findings and overall pass/fail status.
+    """
+    from framelint import __version__
+
+    cfg = _coerce_config(config)
+    df = load_data(source)
+
+    findings = run_all_checks(df, cfg)
+    if baseline is not None:
+        findings.extend(compute_drift(df, baseline, cfg))
+        findings.sort(key=lambda f: f.severity.rank, reverse=True)
+
+    return Report(
+        n_rows=len(df),
+        n_columns=df.shape[1],
+        columns=[str(c) for c in df.columns],
+        findings=findings,
+        fail_on=cfg.fail_on,
+        framelint_version=__version__,
+    )
+
+
+def save_baseline(source: DataSource, path: str | Path) -> Path:
+    """Capture a baseline schema from ``source`` and write it to ``path``.
+
+    Args:
+        source: A :class:`pandas.DataFrame`, or a path to a CSV/Parquet file.
+        path: Destination path for the baseline JSON.
+
+    Returns:
+        The path the baseline was written to.
+    """
+    df = load_data(source)
+    return _save_baseline(df, path)
+
+
+def _coerce_config(config: Config | dict[str, Any] | None) -> Config:
+    """Turn the public ``config`` argument into a concrete :class:`Config`."""
+    if config is None:
+        return Config()
+    if isinstance(config, Config):
+        return config
+    return Config.from_dict(config)

framelint/baseline.py

@@ -0,0 +1,206 @@
+"""Baseline schema capture and schema-drift detection.
+
+A baseline records a dataset's structure and a few key per-column statistics to
+a JSON file. A later dataset can then be compared against it to surface added or
+removed columns, dtype changes, and large distribution shifts.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Union
+
+import pandas as pd
+
+from framelint.config import Config
+from framelint.models import Finding, Severity
+
+BASELINE_VERSION = 1
+
+BaselineLike = Union[str, Path, dict[str, Any]]
+
+
+def build_baseline(df: pd.DataFrame) -> dict[str, Any]:
+    """Capture a JSON-serializable baseline schema from a DataFrame.
+
+    Args:
+        df: The reference dataset.
+
+    Returns:
+        A dict describing the schema: row count and, per column, dtype, null
+        rate, distinct count, and (for numeric columns) summary statistics.
+    """
+    n_rows = len(df)
+    columns: dict[str, Any] = {}
+    for col in df.columns:
+        series = df[col]
+        null_count = int(series.isna().sum())
+        entry: dict[str, Any] = {
+            "dtype": str(series.dtype),
+            "null_rate": round(null_count / n_rows, 6) if n_rows else 0.0,
+            "n_unique": int(series.nunique(dropna=True)),
+        }
+        if pd.api.types.is_numeric_dtype(series) and not pd.api.types.is_bool_dtype(series):
+            values = series.dropna().astype(float)
+            if not values.empty:
+                entry["numeric"] = {
+                    "mean": round(float(values.mean()), 6),
+                    "std": round(float(values.std(ddof=0)), 6),
+                    "min": round(float(values.min()), 6),
+                    "max": round(float(values.max()), 6),
+                }
+        columns[str(col)] = entry
+    return {
+        "framelint_baseline_version": BASELINE_VERSION,
+        "n_rows": n_rows,
+        "columns": columns,
+    }
+
+
+def save_baseline(df: pd.DataFrame, path: str | Path) -> Path:
+    """Write a baseline schema for ``df`` to ``path`` as JSON.
+
+    Args:
+        df: The reference dataset.
+        path: Destination file path.
+
+    Returns:
+        The path written to.
+    """
+    baseline = build_baseline(df)
+    out = Path(path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(baseline, indent=2), encoding="utf-8")
+    return out
+
+
+def load_baseline(baseline: BaselineLike) -> dict[str, Any]:
+    """Load a baseline from a path or accept an already-loaded dict.
+
+    Args:
+        baseline: A path to a baseline JSON file, or a baseline dict.
+
+    Returns:
+        The baseline dict.
+
+    Raises:
+        FileNotFoundError: If a path is given but does not exist.
+        ValueError: If the file does not contain a valid framelint baseline.
+    """
+    if isinstance(baseline, dict):
+        data = baseline
+    else:
+        path = Path(baseline)
+        if not path.is_file():
+            raise FileNotFoundError(f"baseline file not found: {path}")
+        data = json.loads(path.read_text(encoding="utf-8"))
+    if "columns" not in data or "framelint_baseline_version" not in data:
+        raise ValueError("invalid baseline: missing required keys")
+    return data
+
+
+def compute_drift(df: pd.DataFrame, baseline: BaselineLike, config: Config) -> list[Finding]:
+    """Compare a dataset against a baseline and report schema drift.
+
+    Args:
+        df: The new dataset to compare.
+        baseline: The reference baseline (path or dict).
+        config: Thresholds controlling distribution-shift sensitivity.
+
+    Returns:
+        A list of drift findings (added/removed columns, dtype changes, null-rate
+        increases, and numeric mean shifts).
+    """
+    data = load_baseline(baseline)
+    base_columns: dict[str, Any] = data["columns"]
+    findings: list[Finding] = []
+    n_rows = len(df)
+    current = set(map(str, df.columns))
+    baseline_cols = set(base_columns)
+
+    for removed in sorted(baseline_cols - current):
+        findings.append(
+            Finding(
+                check="drift",
+                severity=Severity.ERROR,
+                message=f"Column '{removed}' present in baseline is missing from the data.",
+                column=removed,
+                details={"kind": "removed_column"},
+            )
+        )
+    for added in sorted(current - baseline_cols):
+        findings.append(
+            Finding(
+                check="drift",
+                severity=Severity.WARNING,
+                message=f"Column '{added}' is new relative to the baseline.",
+                column=added,
+                details={"kind": "added_column"},
+            )
+        )
+
+    for col in sorted(baseline_cols & current):
+        base = base_columns[col]
+        series = df[col]
+        findings.extend(_column_drift(col, series, base, n_rows, config))
+    return findings
+
+
+def _column_drift(
+    col: str, series: pd.Series, base: dict[str, Any], n_rows: int, config: Config
+) -> list[Finding]:
+    """Report drift for a single column present in both datasets."""
+    findings: list[Finding] = []
+    current_dtype = str(series.dtype)
+    if current_dtype != base["dtype"]:
+        findings.append(
+            Finding(
+                check="drift",
+                severity=Severity.WARNING,
+                message=f"Column '{col}' dtype changed: {base['dtype']} -> {current_dtype}.",
+                column=col,
+                details={"kind": "dtype_change", "from": base["dtype"], "to": current_dtype},
+            )
+        )
+
+    null_rate = float(series.isna().sum()) / n_rows if n_rows else 0.0
+    increase = null_rate - float(base["null_rate"])
+    if increase > config.drift_null_rate_increase:
+        findings.append(
+            Finding(
+                check="drift",
+                severity=Severity.WARNING,
+                message=f"Column '{col}' null rate rose from {base['null_rate']:.1%} "
+                f"to {null_rate:.1%}.",
+                column=col,
+                details={
+                    "kind": "null_rate_increase",
+                    "from": base["null_rate"],
+                    "to": round(null_rate, 6),
+                },
+            )
+        )
+
+    base_numeric = base.get("numeric")
+    if base_numeric and pd.api.types.is_numeric_dtype(series):
+        values = series.dropna().astype(float)
+        std = float(base_numeric["std"])
+        if not values.empty and std > 0:
+            shift = abs(float(values.mean()) - float(base_numeric["mean"])) / std
+            if shift > config.drift_mean_shift:
+                findings.append(
+                    Finding(
+                        check="drift",
+                        severity=Severity.WARNING,
+                        message=f"Column '{col}' mean shifted {shift:.1f} std from baseline.",
+                        column=col,
+                        details={
+                            "kind": "mean_shift",
+                            "shift_in_std": round(shift, 6),
+                            "baseline_mean": base_numeric["mean"],
+                            "current_mean": round(float(values.mean()), 6),
+                        },
+                    )
+                )
+    return findings