flasheda 0.4.0__tar.gz → 0.5.0__tar.gz

{flasheda-0.4.0 → flasheda-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flasheda
-Version: 0.4.0
+Version: 0.5.0
 Summary: Fixed-time EDA for any dataset size, plus validated cleaning, feature and training code for pandas, polars, DuckDB and PySpark.
 Home-page: https://github.com/sjapanjots/Flash-EDA
 Author: Japanjot Singh
@@ -9,7 +9,7 @@ License: MIT
 Project-URL: Source, https://github.com/sjapanjots/Flash-EDA
 Project-URL: Changelog, https://github.com/sjapanjots/Flash-EDA/blob/main/CHANGELOG.md
 Project-URL: Issues, https://github.com/sjapanjots/Flash-EDA/issues
-Keywords: eda,data-cleaning,pandas,polars,duckdb,pyspark,code-generation,feature-engineering,automl
+Keywords: eda,data-cleaning,pandas,polars,duckdb,pyspark,code-generation,feature-engineering,automl,data-quality,data-drift
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Science/Research
 Classifier: Intended Audience :: Developers
@@ -35,17 +35,23 @@ Requires-Dist: pyspark>=3.4; extra == "spark"
 Provides-Extra: ml
 Requires-Dist: scikit-learn>=1.2; extra == "ml"
 Requires-Dist: pyarrow>=12; extra == "ml"
+Provides-Extra: sql
+Requires-Dist: sqlalchemy>=1.4; extra == "sql"
+Requires-Dist: pyarrow>=12; extra == "sql"
 Provides-Extra: all
 Requires-Dist: duckdb>=0.10; extra == "all"
 Requires-Dist: polars>=0.20; extra == "all"
 Requires-Dist: pyarrow>=12; extra == "all"
 Requires-Dist: scikit-learn>=1.2; extra == "all"
+Requires-Dist: sqlalchemy>=1.4; extra == "all"
 Provides-Extra: dev
 Requires-Dist: pytest>=7; extra == "dev"
 Requires-Dist: duckdb>=0.10; extra == "dev"
 Requires-Dist: polars>=0.20; extra == "dev"
 Requires-Dist: pyarrow>=12; extra == "dev"
 Requires-Dist: scikit-learn>=1.2; extra == "dev"
+Requires-Dist: pandera; extra == "dev"
+Requires-Dist: sqlalchemy>=1.4; extra == "dev"
 Requires-Dist: build; extra == "dev"
 Requires-Dist: twine; extra == "dev"
 Dynamic: author
@@ -72,7 +78,7 @@ Dynamic: summary
 [![Tests](https://github.com/sjapanjots/Flash-EDA/actions/workflows/tests.yml/badge.svg)](https://github.com/sjapanjots/Flash-EDA/actions/workflows/tests.yml)
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](MIT.License)
 
-**Current version: 0.4.0** · [Changelog](CHANGELOG.md)
+**Current version: 0.5.0** · [Changelog](CHANGELOG.md)
 
 Analyse any dataset in a few seconds, then get **validated cleaning, feature and training code** that runs on the full data, in the engine that fits its size: pandas, polars, DuckDB or PySpark.
 
@@ -84,6 +90,7 @@ FlashEDA analyses a fixed-size random sample, so analysis time barely changes be
 pip install flasheda            # core
 pip install "flasheda[fast]"    # + polars, DuckDB, pyarrow (recommended for big files)
 pip install "flasheda[ml]"      # + scikit-learn, for flasheda.train
+pip install "flasheda[sql]"     # + SQLAlchemy, for database tables
 pip install "flasheda[spark]"   # + PySpark
 pip install "flasheda[all]"     # everything except PySpark
 ```
@@ -106,6 +113,18 @@ code.save("clean_orders.py")       # then: python clean_orders.py
 result = fe.train("orders.csv", target="churn", engine="polars")
 result.show()                      # leaderboard
 result.save("prep.py", "train.py") # python prep.py && python train.py → model.joblib
+
+# 4. Guard: learn rules from good data, check every new file
+contract = fe.analyze("orders_jan.csv").contract()
+fe.check("orders_feb.csv", contract).show()
+open("check.py", "w").write(contract.script())   # python check.py new.csv → exit 1 on failure
+
+# 5. Compare: what changed between two datasets
+fe.compare("orders_jan.csv", "orders_feb.csv").show()
+
+# Databases work like files
+src = fe.sql("postgresql://user:pass@host/shop", table="orders")
+fe.analyze(src, target="churn")
 ```
 
 ## What it does
@@ -120,6 +139,9 @@ result.save("prep.py", "train.py") # python prep.py && python train.py → model
 | **Code generation** | one cleaning plan → pandas, polars (lazy, streaming), DuckDB SQL (larger than memory) or PySpark |
 | **Features** | date parts, one-hot / frequency encoding, ratios that beat their inputs |
 | **Models** | baseline, linear, random forest, gradient boosting compared with the right metric and split; training script for the winner |
+| **Data contracts** | rules learned from good data; full-data checks; standalone check script for CI; pandera export |
+| **Drift** | schema changes and per-column drift (PSI, KS, new categories, null-rate change) between two datasets |
+| **Databases** | any SQLAlchemy URL; sampling runs in the database; passwords never stored |
 | **Validation** | generated code is run on the sample and compared with a reference result before you get it |
 
 ### Cleaning steps
@@ -141,6 +163,13 @@ Each step in the generated code has a comment saying why it was added:
 
 The metric and split follow the target: PR-AUC for imbalanced binary targets, MAE for skewed regression targets, time-based splits when there's a date column, and so on.
 
+### Data contracts and drift
+A **contract** describes what good data looks like, learned from a sample with tolerances so normal variation passes:
+type, missing %, 1st/99th percentile range (catches unit changes like ×100), no negatives, allowed values, uniqueness of IDs.
+`fe.check()` checks every row of a file in one DuckDB pass; `contract.script()` writes the same check as a standalone script that exits with code 1 on failure, ready for a pipeline or CI job.
+
+`fe.compare(old, new)` reports added/removed columns, type changes and how much each column shifted (PSI: < 0.1 stable, 0.1–0.25 moderate, ≥ 0.25 major). Dates and IDs are shown as range shifts, since they always move.
+
 ### Engine choice
 
 | Rows | Engine |
@@ -163,12 +192,16 @@ Measured on synthetic messy data:
 | Run the generated polars script on 1M rows | 0.9 s |
 | `fe.train()` on a 1M-row CSV (plan, features, 4 models, checks) | 6.7 s |
 | Generated `prep.py` + `train.py` on 1M rows | 1.3 s + 20 s |
+| Learn a contract from 1M rows / run the check script on 1M rows | 0.8 s / 0.6–0.9 s |
+| Compare two 1M-row files | 0.6 s |
+| Analyse a 1M-row SQLite table (5,000 rows transferred) | 0.3 s |
 
 ## Examples
 
 - [`examples/phase1_demo.py`](examples/phase1_demo.py): engine choice and code generation at 0.1M / 1M / 5M rows
 - [`examples/phase2_demo.py`](examples/phase2_demo.py): target analysis and cleaning on a 1M-row churn CSV
 - [`examples/phase3_demo.py`](examples/phase3_demo.py): features, model comparison and training on 1M rows
+- [`examples/phase4_demo.py`](examples/phase4_demo.py): contracts, check script, drift and a database on 1M rows
 
 ## Development
 

{flasheda-0.4.0 → flasheda-0.5.0}/README.md

@@ -7,7 +7,7 @@
 [![Tests](https://github.com/sjapanjots/Flash-EDA/actions/workflows/tests.yml/badge.svg)](https://github.com/sjapanjots/Flash-EDA/actions/workflows/tests.yml)
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](MIT.License)
 
-**Current version: 0.4.0** · [Changelog](CHANGELOG.md)
+**Current version: 0.5.0** · [Changelog](CHANGELOG.md)
 
 Analyse any dataset in a few seconds, then get **validated cleaning, feature and training code** that runs on the full data, in the engine that fits its size: pandas, polars, DuckDB or PySpark.
 
@@ -19,6 +19,7 @@ FlashEDA analyses a fixed-size random sample, so analysis time barely changes be
 pip install flasheda            # core
 pip install "flasheda[fast]"    # + polars, DuckDB, pyarrow (recommended for big files)
 pip install "flasheda[ml]"      # + scikit-learn, for flasheda.train
+pip install "flasheda[sql]"     # + SQLAlchemy, for database tables
 pip install "flasheda[spark]"   # + PySpark
 pip install "flasheda[all]"     # everything except PySpark
 ```
@@ -41,6 +42,18 @@ code.save("clean_orders.py")       # then: python clean_orders.py
 result = fe.train("orders.csv", target="churn", engine="polars")
 result.show()                      # leaderboard
 result.save("prep.py", "train.py") # python prep.py && python train.py → model.joblib
+
+# 4. Guard: learn rules from good data, check every new file
+contract = fe.analyze("orders_jan.csv").contract()
+fe.check("orders_feb.csv", contract).show()
+open("check.py", "w").write(contract.script())   # python check.py new.csv → exit 1 on failure
+
+# 5. Compare: what changed between two datasets
+fe.compare("orders_jan.csv", "orders_feb.csv").show()
+
+# Databases work like files
+src = fe.sql("postgresql://user:pass@host/shop", table="orders")
+fe.analyze(src, target="churn")
 ```
 
 ## What it does
@@ -55,6 +68,9 @@ result.save("prep.py", "train.py") # python prep.py && python train.py → model
 | **Code generation** | one cleaning plan → pandas, polars (lazy, streaming), DuckDB SQL (larger than memory) or PySpark |
 | **Features** | date parts, one-hot / frequency encoding, ratios that beat their inputs |
 | **Models** | baseline, linear, random forest, gradient boosting compared with the right metric and split; training script for the winner |
+| **Data contracts** | rules learned from good data; full-data checks; standalone check script for CI; pandera export |
+| **Drift** | schema changes and per-column drift (PSI, KS, new categories, null-rate change) between two datasets |
+| **Databases** | any SQLAlchemy URL; sampling runs in the database; passwords never stored |
 | **Validation** | generated code is run on the sample and compared with a reference result before you get it |
 
 ### Cleaning steps
@@ -76,6 +92,13 @@ Each step in the generated code has a comment saying why it was added:
 
 The metric and split follow the target: PR-AUC for imbalanced binary targets, MAE for skewed regression targets, time-based splits when there's a date column, and so on.
 
+### Data contracts and drift
+A **contract** describes what good data looks like, learned from a sample with tolerances so normal variation passes:
+type, missing %, 1st/99th percentile range (catches unit changes like ×100), no negatives, allowed values, uniqueness of IDs.
+`fe.check()` checks every row of a file in one DuckDB pass; `contract.script()` writes the same check as a standalone script that exits with code 1 on failure, ready for a pipeline or CI job.
+
+`fe.compare(old, new)` reports added/removed columns, type changes and how much each column shifted (PSI: < 0.1 stable, 0.1–0.25 moderate, ≥ 0.25 major). Dates and IDs are shown as range shifts, since they always move.
+
 ### Engine choice
 
 | Rows | Engine |
@@ -98,12 +121,16 @@ Measured on synthetic messy data:
 | Run the generated polars script on 1M rows | 0.9 s |
 | `fe.train()` on a 1M-row CSV (plan, features, 4 models, checks) | 6.7 s |
 | Generated `prep.py` + `train.py` on 1M rows | 1.3 s + 20 s |
+| Learn a contract from 1M rows / run the check script on 1M rows | 0.8 s / 0.6–0.9 s |
+| Compare two 1M-row files | 0.6 s |
+| Analyse a 1M-row SQLite table (5,000 rows transferred) | 0.3 s |
 
 ## Examples
 
 - [`examples/phase1_demo.py`](examples/phase1_demo.py): engine choice and code generation at 0.1M / 1M / 5M rows
 - [`examples/phase2_demo.py`](examples/phase2_demo.py): target analysis and cleaning on a 1M-row churn CSV
 - [`examples/phase3_demo.py`](examples/phase3_demo.py): features, model comparison and training on 1M rows
+- [`examples/phase4_demo.py`](examples/phase4_demo.py): contracts, check script, drift and a database on 1M rows
 
 ## Development
 

{flasheda-0.4.0 → flasheda-0.5.0}/flasheda/__init__.py

@@ -10,21 +10,26 @@ import pandas as pd
 from . import codegen, engine
 from .clean import clean
 from .modeling import train
-from .engine import available_engines, choose_engine, inspect_source, sample, get_sample_size, set_sample_size
+from .contract import check
+from .compare import compare
+from .engine import available_engines, choose_engine, inspect_source, sample, get_sample_size, set_sample_size, sql
 from .analyzers import overview, missing, numeric, categorical, correlation
 from .analyzers import target as target_analyzer
 from .report import EDAReport
 
-__version__ = "0.4.0"
+__version__ = "0.5.0"
 __all__ = [
     "analyze",
     "available_engines",
+    "check",
     "clean",
+    "compare",
     "codegen",
     "engine",
     "get_sample_size",
     "inspect_source",
     "set_sample_size",
+    "sql",
     "train",
     "EDAReport",
 ]
@@ -42,7 +47,8 @@ def analyze(
     target: Optional[str] = None,
 ) -> EDAReport:
     """
-    Analyse a fixed-size sample of `source` (DataFrame, CSV/TSV or Parquet path).
+    Analyse a fixed-size sample of `source`: a DataFrame, a CSV/TSV or Parquet
+    path, or a database table/query from flasheda.sql().
 
     `target` names the column you want to predict; it adds task detection,
     class balance, per-feature predictive power and leakage checks.
@@ -121,7 +127,10 @@ def analyze(
         warnings=warnings,
         target=tgt,
         source=str(info.path) if info.path else None,
-        recommended_engine=choose_engine(info.n_rows, prefer=engine),
+        sql_source=info.sql,
+        # database sources are read with pandas (or exported with .to_parquet())
+        recommended_engine=choose_engine(
+            info.n_rows, prefer=engine or ("pandas" if info.sql is not None else None)),
         sample=sampled,
     )
 

{flasheda-0.4.0 → flasheda-0.5.0}/flasheda/codegen/__init__.py

@@ -58,10 +58,12 @@ def generate(
     output: str = "cleaned.parquet",
     sample: Optional[pd.DataFrame] = None,
     columns: Optional[List[str]] = None,
+    sql: Any = None,
 ) -> GeneratedCode:
+    """`sql` is a flasheda.sql() source to read from instead of a file."""
     if columns is None and sample is not None:
         columns = [str(c) for c in sample.columns]
-    code = get_writer(engine).render(plan, source, output, columns)
+    code = get_writer(engine).render(plan, source, output, columns, sql=sql)
     return GeneratedCode(engine, code, plan, sample)
 
 

{flasheda-0.4.0 → flasheda-0.5.0}/flasheda/codegen/writers/base.py

@@ -14,7 +14,7 @@ from __future__ import annotations
 import textwrap
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import List, Optional
+from typing import Any, List, Optional
 
 from ..steps import Plan, Step
 
@@ -23,6 +23,8 @@ class Writer(ABC):
     engine: str = ""
     install_hint: str = ""
     columns: Optional[List[str]] = None
+    sql: Any = None
+    reads_sql = False           # writers that can load from a database URL
 
     def render(
         self,
@@ -30,9 +32,17 @@ class Writer(ABC):
         source: Optional[str],
         output: str,
         columns: Optional[List[str]] = None,
+        sql: Any = None,
     ) -> str:
-        """`columns` are the source's column names, when known."""
+        """`columns` are the source's column names, when known; `sql` a database source."""
         self.columns = list(columns) if columns is not None else None
+        self.sql = sql
+        if sql is not None and not self.reads_sql:
+            raise ValueError(
+                f"The {self.engine} writer can't read from a database directly. Use engine='pandas', "
+                "or export first: flasheda.sql(...).to_parquet('data.parquet'), then analyze that file.")
+        if sql is not None:
+            source = sql.display
         parts = [
             self._docstring(plan, source),
             self.imports(),

{flasheda-0.4.0 → flasheda-0.5.0}/flasheda/codegen/writers/pandas_writer.py

@@ -13,6 +13,7 @@ from .base import Writer, source_format
 class PandasWriter(Writer):
     engine = "pandas"
     install_hint = "pandas, pyarrow"
+    reads_sql = True
 
     def imports(self) -> str:
         return "import numpy as np\nimport pandas as pd"
@@ -73,6 +74,8 @@ class PandasWriter(Writer):
         raise TypeError(f"Unsupported step {step!r}")
 
     def main(self, source: Optional[str]) -> str:
+        if self.sql is not None:
+            return self._main_sql()
         fmt = source_format(source)
         if fmt == "parquet":
             load = "pd.read_parquet(SOURCE)"
@@ -85,3 +88,19 @@ class PandasWriter(Writer):
             "    df.to_parquet(OUTPUT, index=False)\n"
             '    print(f"Wrote {len(df):,} rows -> {OUTPUT}")'
         )
+
+    def _main_sql(self) -> str:
+        from ...engine.sql import URL_ENV_VAR
+        query = f"SELECT * FROM {self.sql.from_sql()}"
+        return (
+            "def main() -> None:\n"
+            "    import os\n"
+            "    from sqlalchemy import create_engine, text\n\n"
+            f"    # The database URL is read from ${URL_ENV_VAR} so no password is stored here.\n"
+            f"    url = os.environ[{URL_ENV_VAR!r}]\n"
+            "    with create_engine(url).connect() as con:\n"
+            f"        df = pd.read_sql(text({query!r}), con)\n"
+            "    df = transform(df)\n"
+            "    df.to_parquet(OUTPUT, index=False)\n"
+            '    print(f"Wrote {len(df):,} rows -> {OUTPUT}")'
+        )

flasheda-0.5.0/flasheda/compare/__init__.py

@@ -0,0 +1,224 @@
+"""
+flasheda.compare
+~~~~~~~~~~~~~~~~
+What changed between two datasets (last month vs this month, train vs
+test), measured on fixed-size samples so it takes seconds at any size:
+
+    drift = flasheda.compare("orders_jan.csv", "orders_feb.csv")
+    drift.show()
+
+  schema     columns added / removed, type changes
+  per column PSI (population stability index) on reference quantile bins or
+             categories; KS test for numbers; new categories; null-rate change
+  dates, IDs reported as a range shift (they always move), not as drift
+  severity   PSI < 0.1 stable · 0.1–0.25 moderate · ≥ 0.25 major (the usual
+             credit-risk thresholds); a null rate moving ≥ 10 points is at
+             least moderate
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+
+from ..contract.rules import _kind
+from ..engine import inspect_source, sample
+
+__all__ = ["DriftReport", "compare", "psi"]
+
+PSI_MODERATE = 0.10
+PSI_MAJOR = 0.25
+NULL_SHIFT_POINTS = 10.0
+BINS = 10
+TOP_CATEGORIES = 20
+EPS = 1e-4
+HIGH_CARDINALITY = 0.5     # unique / rows above this: IDs, free text
+
+
+@dataclass
+class DriftReport:
+    rows_reference: int
+    rows_current: int
+    added_columns: List[str]
+    removed_columns: List[str]
+    type_changes: List[Dict[str, str]]
+    columns: List[Dict[str, Any]] = field(default_factory=list)
+
+    @property
+    def drifted(self) -> List[Dict[str, Any]]:
+        return [c for c in self.columns if c["severity"] != "stable"]
+
+    @property
+    def ok(self) -> bool:
+        """No schema changes and no major drift."""
+        return (not self.added_columns and not self.removed_columns and not self.type_changes
+                and all(c["severity"] != "major" for c in self.columns))
+
+    def __getitem__(self, column: str) -> Dict[str, Any]:
+        for c in self.columns:
+            if c["column"] == column:
+                return c
+        raise KeyError(column)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {"ok": self.ok, "rows_reference": self.rows_reference,
+                "rows_current": self.rows_current, "added_columns": self.added_columns,
+                "removed_columns": self.removed_columns, "type_changes": self.type_changes,
+                "columns": self.columns}
+
+    def show(self) -> None:
+        from rich import box
+        from rich.console import Console
+        from rich.table import Table
+
+        console = Console()
+        console.print(f"[bold]Drift[/bold]: {self.rows_reference:,} → {self.rows_current:,} rows  "
+                      + ("[green]no major drift[/]" if self.ok else "[red]changes found[/]"))
+        if self.added_columns:
+            console.print(f"  [yellow]added columns:[/] {', '.join(self.added_columns)}")
+        if self.removed_columns:
+            console.print(f"  [red]removed columns:[/] {', '.join(self.removed_columns)}")
+        for t in self.type_changes:
+            console.print(f"  [red]type change:[/] {t['column']}: {t['reference']} → {t['current']}")
+        table = Table(box=box.SIMPLE)
+        for h in ("column", "PSI", "severity", "null % (ref → cur)", "detail"):
+            table.add_column(h)
+        colors = {"stable": "green", "moderate": "yellow", "major": "red"}
+        for c in self.columns:
+            table.add_row(c["column"], f"{c['psi']:.3f}", f"[{colors[c['severity']]}]{c['severity']}[/]",
+                          f"{c['null_pct_reference']} → {c['null_pct_current']}", c["detail"])
+        console.print(table)
+
+    def __repr__(self) -> str:
+        return (f"DriftReport(ok={self.ok}, drifted={len(self.drifted)}/{len(self.columns)}, "
+                f"added={len(self.added_columns)}, removed={len(self.removed_columns)})")
+
+
+def compare(reference: Any, current: Any, *, n: Optional[int] = None,
+            random_state: int = 42) -> DriftReport:
+    ref_info, cur_info = inspect_source(reference), inspect_source(current)
+    ref = sample(reference, n=n, random_state=random_state)
+    cur = sample(current, n=n, random_state=random_state)
+    ref.columns = [str(c) for c in ref.columns]
+    cur.columns = [str(c) for c in cur.columns]
+
+    common = [c for c in ref.columns if c in cur.columns]
+    report = DriftReport(
+        rows_reference=ref_info.n_rows if ref_info.n_rows is not None else len(ref),
+        rows_current=cur_info.n_rows if cur_info.n_rows is not None else len(cur),
+        added_columns=[c for c in cur.columns if c not in ref.columns],
+        removed_columns=[c for c in ref.columns if c not in cur.columns],
+        type_changes=[],
+    )
+    for col in common:
+        k_ref, k_cur = _kind(ref[col]), _kind(cur[col])
+        if k_ref != k_cur:
+            report.type_changes.append({"column": col, "reference": k_ref, "current": k_cur})
+            continue
+        report.columns.append(_column_drift(col, ref[col], cur[col], k_ref))
+    report.columns.sort(key=lambda c: c["psi"], reverse=True)
+    return report
+
+
+def psi(expected: np.ndarray, actual: np.ndarray) -> float:
+    """Population stability index between two proportion vectors."""
+    e = np.clip(np.asarray(expected, float), EPS, None)
+    a = np.clip(np.asarray(actual, float), EPS, None)
+    return float(np.sum((a - e) * np.log(a / e)))
+
+
+def _column_drift(col: str, ref: pd.Series, cur: pd.Series, kind: str) -> Dict[str, Any]:
+    null_ref = round(float(ref.isna().mean() * 100), 2)
+    null_cur = round(float(cur.isna().mean() * 100), 2)
+    out: Dict[str, Any] = {"column": col, "kind": kind,
+                           "null_pct_reference": null_ref, "null_pct_current": null_cur}
+
+    if kind == "datetime":
+        # Dates move forward by nature, so they're reported as a range shift,
+        # not as drift; only their null rate can raise the severity.
+        r = pd.to_datetime(ref, errors="coerce", format="mixed").dropna()
+        c = pd.to_datetime(cur, errors="coerce", format="mixed").dropna()
+        value = 0.0
+        out["detail"] = (f"{r.min():%Y-%m-%d}…{r.max():%Y-%m-%d} → {c.min():%Y-%m-%d}…{c.max():%Y-%m-%d}"
+                         if len(r) and len(c) else "")
+    elif kind == "integer" and _is_id_like(ref):
+        # New IDs always differ from old ones: report the range, not drift.
+        r, c = ref.dropna(), cur.dropna()
+        value = 0.0
+        out["detail"] = (f"ID-like: {r.min():,}…{r.max():,} → {c.min():,}…{c.max():,}"
+                         if len(r) and len(c) else "ID-like")
+    elif kind in ("number", "integer"):
+        r = pd.to_numeric(ref, errors="coerce")
+        c = pd.to_numeric(cur, errors="coerce")
+        value = _numeric_psi(r, c)
+        out.update(_numeric_details(r.dropna(), c.dropna()))
+    else:
+        value, new_share, new_values = _categorical_psi(ref.astype("string"), cur.astype("string"))
+        values = ref.dropna()
+        if len(values) and values.nunique() / len(values) > HIGH_CARDINALITY:
+            out["detail"] = "mostly unique values: compared by most common values only"
+        else:
+            out["new_values_pct"] = round(new_share * 100, 2)
+            out["new_values"] = new_values
+            out["detail"] = (f"{out['new_values_pct']}% new values, e.g. {', '.join(new_values[:3])}"
+                             if new_values else "")
+    out["psi"] = round(value, 4)
+
+    severity = "major" if value >= PSI_MAJOR else "moderate" if value >= PSI_MODERATE else "stable"
+    if severity == "stable" and abs(null_cur - null_ref) >= NULL_SHIFT_POINTS:
+        severity = "moderate"
+    out["severity"] = severity
+    return out
+
+
+def _is_id_like(s: pd.Series) -> bool:
+    values = s.dropna()
+    return len(values) > 0 and values.nunique() / len(values) > 0.95
+
+
+def _numeric_psi(ref: pd.Series, cur: pd.Series) -> float:
+    r, c = ref.dropna().to_numpy(), cur.dropna().to_numpy()
+    if not len(r) or not len(c):
+        return 0.0
+    edges = np.unique(np.quantile(r, np.linspace(0, 1, BINS + 1)))
+    if len(edges) < 2:                                   # constant reference
+        return psi([1.0, 0.0], [np.mean(c == edges[0]), np.mean(c != edges[0])])
+    edges[0], edges[-1] = -np.inf, np.inf
+    # missing values get their own bin, so a jump in nulls counts as drift
+    e = np.append(np.histogram(r, edges)[0] / len(ref), ref.isna().mean())
+    a = np.append(np.histogram(c, edges)[0] / len(cur), cur.isna().mean())
+    return psi(e, a)
+
+
+def _numeric_details(r: pd.Series, c: pd.Series) -> Dict[str, Any]:
+    from scipy.stats import ks_2samp
+
+    if not len(r) or not len(c):
+        return {"detail": "no values to compare"}
+    ks = ks_2samp(r, c)
+    std = float(r.std()) or 1.0
+    shift = (float(c.median()) - float(r.median())) / std
+    return {
+        "median_reference": float(r.median()),
+        "median_current": float(c.median()),
+        "median_shift_std": round(shift, 3),
+        "ks_statistic": round(float(ks.statistic), 4),
+        "ks_pvalue": float(ks.pvalue),
+        "detail": f"median {float(r.median()):.4g} → {float(c.median()):.4g} ({shift:+.2f} std)",
+    }
+
+
+def _categorical_psi(ref: pd.Series, cur: pd.Series):
+    ref_counts = ref.fillna("<null>").value_counts(normalize=True)
+    cur_counts = cur.fillna("<null>").value_counts(normalize=True)
+    top = list(ref_counts.index[:TOP_CATEGORIES])
+    if "<null>" not in top:
+        top.append("<null>")
+    e = [ref_counts.get(v, 0.0) for v in top] + [1 - ref_counts.reindex(top).fillna(0).sum()]
+    a = [cur_counts.get(v, 0.0) for v in top] + [1 - cur_counts.reindex(top).fillna(0).sum()]
+    seen = set(ref_counts.index) | {"<null>"}      # nulls are reported as null %, not new values
+    new = cur_counts[[v not in seen for v in cur_counts.index]]
+    return psi(e, a), float(new.sum()), [str(v) for v in new.index[:10]]

flasheda-0.5.0/flasheda/contract/__init__.py

@@ -0,0 +1,29 @@
+"""
+flasheda.contract
+~~~~~~~~~~~~~~~~~
+Learn a data contract from a sample, then check new or full data against it:
+
+    report = flasheda.analyze("orders_jan.csv")
+    contract = report.contract()
+    contract.save("orders.contract.json")
+
+    result = flasheda.check("orders_feb.csv", "orders.contract.json")
+    result.show()            # failures, if any
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Union
+
+from .check import CheckResult, Outcome
+from .rules import ColumnRule, Contract, infer_contract
+
+__all__ = ["CheckResult", "ColumnRule", "Contract", "Outcome", "check", "infer_contract"]
+
+
+def check(source: Any, contract: Union[Contract, str, Path]) -> CheckResult:
+    """Check `source` against a Contract or a saved contract JSON file."""
+    if not isinstance(contract, Contract):
+        contract = Contract.load(contract)
+    return contract.check(source)