cleanframe-engine 0.3.0__py3-none-any.whl

cleanframe/types.py

@@ -0,0 +1,119 @@
+"""Shared enums and small value types used across CleanFrame.
+
+Kept dependency-free (stdlib only) so every other module can import it without
+risk of an import cycle.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+class Mode(str, Enum):
+    """How aggressively CleanFrame commits to fixes and how it reacts to problems.
+
+    ``str`` mixin so a plain ``"review"`` string round-trips through configs and
+    YAML transparently (``Mode("review") == "review"``).
+    """
+
+    #: Build the recipe and cleaned frame, quarantine validation failures, and
+    #: surface everything for a human to approve. Nothing is written to disk
+    #: automatically. This is the default.
+    REVIEW = "review"
+
+    #: For unattended pipelines: only fixes at or above a higher confidence
+    #: threshold than ``review`` are planned (see ``planner.MODE_THRESHOLDS``).
+    AUTO = "auto"
+
+    #: Zero tolerance. Validation failures and schema drift raise instead of
+    #: being quarantined or silently dropped, and only the highest-confidence
+    #: fixes are planned at all — lower-confidence proposals are omitted.
+    STRICT = "strict"
+
+    @classmethod
+    def coerce(cls, value: Mode | str) -> Mode:
+        if isinstance(value, cls):
+            return value
+        try:
+            return cls(str(value).strip().lower())
+        except ValueError as exc:
+            from .errors import CleanFrameError
+
+            valid = ", ".join(m.value for m in cls)
+            raise CleanFrameError(
+                f"Unknown mode {value!r}. Expected one of: {valid}"
+            ) from exc
+
+
+class Severity(str, Enum):
+    """Ordered severity for detected issues and validation results."""
+
+    INFO = "info"
+    WARNING = "warning"
+    ERROR = "error"
+
+    @property
+    def rank(self) -> int:
+        return {"info": 0, "warning": 1, "error": 2}[self.value]
+
+    def __lt__(self, other: object) -> bool:
+        if isinstance(other, Severity):
+            return self.rank < other.rank
+        return NotImplemented
+
+
+class LLMExposure(str, Enum):
+    """How much of your data an LLM planner is permitted to see.
+
+    Mirrors the privacy tiers documented in the README. ``NONE`` keeps everything
+    on your machine; higher tiers progressively expand what metadata may leave it,
+    and never include raw cell values unless you explicitly opt into ``SAMPLE``.
+    """
+
+    #: Rules-only. No network calls, nothing leaves the machine.
+    NONE = "none"
+    #: LLM sees column names, dtypes, and value *patterns* (regex sketches) only.
+    METADATA = "metadata"
+    #: LLM additionally sees an anonymized, shuffled sample the caller approved.
+    SAMPLE = "sample"
+
+
+@dataclass
+class Op:
+    """A single transformation step: an op name plus its parameters.
+
+    ``Op`` is the atomic unit of a recipe. It is deliberately dumb — it carries a
+    name and a params dict, and knows how to render itself back to the compact
+    YAML form. The mapping from name to a concrete pandas function lives in the op
+    registry (:mod:`cleanframe.ops`); the mapping to executable Python source lives
+    in :mod:`cleanframe.codegen`. Keeping ``Op`` free of behaviour is what lets the
+    exact same object flow from detector → planner → recipe → executor → codegen.
+
+    The compact form used in recipe YAML is either a bare string (no params) or a
+    single-key mapping ``{name: params}``::
+
+        strip_whitespace                 # -> Op("strip_whitespace", {})
+        {remove_symbols: ["₹", ","]}     # -> Op("remove_symbols", {"symbols": [...]})
+        {cast: float}                    # -> Op("cast", {"to": "float"})
+    """
+
+    name: str
+    params: dict[str, Any] = field(default_factory=dict)
+
+    def to_compact(self) -> Any:
+        """Render to the compact YAML form (bare string when there are no params)."""
+        if not self.params:
+            return self.name
+        return {self.name: dict(self.params)}
+
+    def with_params(self, **updates: Any) -> Op:
+        merged = dict(self.params)
+        merged.update(updates)
+        return Op(self.name, merged)
+
+    def __repr__(self) -> str:  # pragma: no cover - cosmetic
+        if not self.params:
+            return f"Op({self.name!r})"
+        return f"Op({self.name!r}, {self.params!r})"

cleanframe/validate.py

@@ -0,0 +1,363 @@
+"""Post-clean validation with per-rule failure policies.
+
+Validation runs *after* the recipe's transforms. Each rule computes a pass/fail
+mask over a column; the rule's ``on_fail`` decides what happens to failing rows:
+
+``quarantine`` (default)
+    Move the row into a side ``quarantine`` frame with a reason — never silently
+    dropped, never corrupting the clean output.
+``error``
+    Raise :class:`~cleanframe.errors.ValidationFailure`. ``strict`` mode promotes
+    *every* policy to ``error``.
+``warn`` / ``drop`` / ``null``
+    Log only / discard the row / blank just the offending cell.
+
+Checks are pluggable, like detectors::
+
+    @cf.validator("valid_iban")
+    def _(series): return series.isna() | series.str.match(IBAN_RE)
+"""
+
+from __future__ import annotations
+
+import re
+import warnings
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+import numpy as np
+import pandas as pd
+import yaml
+
+from ._util import safe_compile_regex
+from .errors import CleanFrameWarning, RecipeError, ValidationFailure
+from .profile import EMAIL_RE, URL_RE
+from .recipe import ValidationRule
+from .types import Mode
+
+# ---------------------------------------------------------------------------
+# Validator registry (named checks)
+# ---------------------------------------------------------------------------
+#: name -> function(series, **params) -> boolean pass-mask (True = ok).
+VALIDATOR_REGISTRY: dict[str, Callable[..., pd.Series]] = {}
+
+
+def validator(name: str) -> Callable[[Callable], Callable]:
+    """Register a named validation check. The function returns a pass-mask."""
+
+    def decorator(func: Callable) -> Callable:
+        if name in VALIDATOR_REGISTRY:
+            raise ValueError(f"Validator {name!r} is already registered.")
+        VALIDATOR_REGISTRY[name] = func
+        return func
+
+    return decorator
+
+
+def list_validators() -> list[str]:
+    return sorted(VALIDATOR_REGISTRY)
+
+
+_PHONE_DIGITS = re.compile(r"\D")
+
+
+@validator("not_null")
+def _not_null(series: pd.Series) -> pd.Series:
+    return series.notna()
+
+
+@validator("unique")
+def _unique(series: pd.Series) -> pd.Series:
+    duplicated = series.duplicated(keep=False) & series.notna()
+    return ~duplicated
+
+
+@validator("valid_email")
+def _valid_email(series: pd.Series) -> pd.Series:
+    def ok(v: Any) -> bool:
+        return bool(EMAIL_RE.match(str(v).strip().lower()))
+
+    return series.isna() | series.map(ok)
+
+
+@validator("valid_url")
+def _valid_url(series: pd.Series) -> pd.Series:
+    return series.isna() | series.map(lambda v: bool(URL_RE.match(str(v).strip())))
+
+
+@validator("valid_phone")
+def _valid_phone(series: pd.Series) -> pd.Series:
+    def ok(v: Any) -> bool:
+        return 7 <= len(_PHONE_DIGITS.sub("", str(v))) <= 15
+
+    return series.isna() | series.map(ok)
+
+
+# ---------------------------------------------------------------------------
+# Expression checks (comparisons / membership / regex)
+# ---------------------------------------------------------------------------
+_CMP_RE = re.compile(r"^(>=|<=|==|!=|>|<)\s*(-?\d+(?:\.\d+)?)$")
+_CMP_OPS: dict[str, Callable[[Any, float], Any]] = {
+    ">=": lambda s, t: s >= t,
+    "<=": lambda s, t: s <= t,
+    ">": lambda s, t: s > t,
+    "<": lambda s, t: s < t,
+    "==": lambda s, t: s == t,
+    "!=": lambda s, t: s != t,
+}
+
+
+def _comparison_mask(series: pd.Series, op: str, threshold: float) -> pd.Series:
+    numeric = pd.to_numeric(series, errors="coerce")
+    orig_na = series.isna()
+    satisfies = _CMP_OPS[op](numeric, threshold)
+    satisfies = satisfies.fillna(False).astype(bool)
+    return orig_na | (numeric.notna() & satisfies)
+
+
+def _membership_values(rule: ValidationRule) -> list[Any]:
+    values = rule.params.get("values")
+    if values is not None:
+        return [values] if isinstance(values, str) else list(values)
+    rest = rule.check[2:].strip()  # drop leading "in"
+    # BaseLoader keeps every scalar a string: YAML 1.1 would read `in [Yes, No]`
+    # as booleans and then reject every literal "Yes"/"No" in the data.
+    parsed = yaml.load(rest, Loader=yaml.BaseLoader) if rest else []
+    return list(parsed) if isinstance(parsed, (list, tuple)) else [parsed]
+
+
+#: YAML 1.1 reads a bare ``Yes``/``No``/``On``/``Off`` as a boolean, so a rule written
+#: ``values: [Yes, No]`` arrives here as ``[True, False]``. Accept every spelling of
+#: the boolean rather than rejecting the literal text the data actually holds.
+_BOOL_SPELLINGS: dict[bool, tuple[str, ...]] = {
+    True: ("True", "true", "TRUE", "Yes", "yes", "YES", "On", "on", "Y", "y", "1"),
+    False: ("False", "false", "FALSE", "No", "no", "NO", "Off", "off", "N", "n", "0"),
+}
+
+
+def _membership_mask(series: pd.Series, values: list[Any]) -> pd.Series:
+    as_str: set[str] = set()
+    for value in values:
+        if isinstance(value, bool):
+            as_str.update(_BOOL_SPELLINGS[value])
+        else:
+            as_str.add(str(value))
+    return series.isna() | series.isin(values) | series.astype(str).isin(as_str)
+
+
+def _regex_mask(series: pd.Series, pattern: str) -> pd.Series:
+    try:
+        compiled = safe_compile_regex(pattern)
+    except ValueError as exc:
+        raise RecipeError(str(exc)) from exc
+    return series.isna() | series.map(lambda v: bool(compiled.search(str(v))))
+
+
+_EXPRESSION_PREFIXES = ("in ", "in[", "matches", "regex")
+
+
+def check_is_known(check: str) -> None:
+    """Raise :class:`RecipeError` for a check that is neither registered nor an expression.
+
+    Called at recipe *load* time so a typo like ``valid_emial`` fails before the
+    recipe is applied to production data.
+    """
+    text = str(check).strip()
+    if not text:
+        raise RecipeError("A validation rule needs a non-empty 'check'.")
+    if text in VALIDATOR_REGISTRY or _CMP_RE.match(text) or text == "in":
+        return
+    if text.startswith(_EXPRESSION_PREFIXES):
+        return
+    raise RecipeError(
+        f"Unknown validation check {check!r}. Known: {', '.join(list_validators())}, "
+        "comparisons (>= 0), 'in [...]', 'matches: <regex>'. Register a custom check "
+        "with @cleanframe.validator before loading the recipe."
+    )
+
+
+def pass_mask(rule: ValidationRule, series: pd.Series) -> pd.Series:
+    """Compute the boolean pass-mask (True = passes) for ``rule`` over ``series``."""
+    check = rule.check.strip()
+    if check in VALIDATOR_REGISTRY:
+        try:
+            mask = VALIDATOR_REGISTRY[check](series, **rule.params)
+        except TypeError as exc:
+            raise RecipeError(
+                f"Validation {check!r} on column {rule.column!r} does not accept "
+                f"parameter(s) {sorted(rule.params)}: {exc}"
+            ) from exc
+        return mask.astype(bool)
+
+    cmp = _CMP_RE.match(check)
+    if cmp:
+        return _comparison_mask(series, cmp.group(1), float(cmp.group(2))).astype(bool)
+
+    if check == "in" or check.startswith("in ") or check.startswith("in["):
+        return _membership_mask(series, _membership_values(rule)).astype(bool)
+
+    if check.startswith("matches") or check.startswith("regex"):
+        pattern = rule.params.get("pattern") or re.sub(r"^(matches|regex):?\s*", "", check)
+        return _regex_mask(series, pattern).astype(bool)
+
+    raise RecipeError(
+        f"Unknown validation check {check!r}. Known: {', '.join(list_validators())}, "
+        "comparisons (>= 0), 'in [...]', 'matches: <regex>'."
+    )
+
+
+# ---------------------------------------------------------------------------
+# Results + application
+# ---------------------------------------------------------------------------
+@dataclass
+class ValidationResult:
+    column: str
+    check: str
+    on_fail: str
+    n_failed: int
+    failed_row_ids: list[int]
+    found: bool = True
+
+    @property
+    def passed(self) -> bool:
+        return self.n_failed == 0
+
+
+@dataclass
+class ValidationOutcome:
+    dataframe: pd.DataFrame
+    quarantine: pd.DataFrame
+    results: list[ValidationResult]
+    removed_rows: list[tuple[int, str]] = field(default_factory=list)
+    log: list[str] = field(default_factory=list)
+
+
+QUARANTINE_REASON_COL = "_cf_quarantine_reason"
+
+
+def _reason_column(df: pd.DataFrame) -> str:
+    """A free name for the quarantine reason, so a user column is never overwritten."""
+    name = QUARANTINE_REASON_COL
+    suffix = 2
+    while name in df.columns:
+        name = f"{QUARANTINE_REASON_COL}_{suffix}"
+        suffix += 1
+    return name
+
+
+def evaluate(rule: ValidationRule, df: pd.DataFrame) -> ValidationResult:
+    if rule.column not in df.columns:
+        return ValidationResult(rule.column, rule.check, rule.on_fail, 0, [], found=False)
+    series = df[rule.column]
+    ok = pass_mask(rule, series)
+    fail_mask = ~ok.to_numpy()
+    failed_ids = [int(i) for i in series.index[fail_mask]]
+    return ValidationResult(rule.column, rule.check, rule.on_fail, len(failed_ids), failed_ids)
+
+
+def _effective_policy(on_fail: str, mode: Mode) -> str:
+    # strict = zero tolerance: ANY failure raises, including drop/null which would
+    # otherwise silently discard rows or blank cells — the opposite of "fail loud".
+    if mode is Mode.STRICT and on_fail != "error":
+        return "error"
+    return on_fail
+
+
+def apply_validations(
+    df: pd.DataFrame, rules: list[ValidationRule], mode: Mode | str = Mode.REVIEW
+) -> ValidationOutcome:
+    """Evaluate all rules against a snapshot, then apply their failure policies."""
+    mode = Mode.coerce(mode)
+    work = df.copy()
+    results = [evaluate(rule, work) for rule in rules]
+
+    error_failures: list[ValidationResult] = []
+    null_actions: list[tuple[str, list[int]]] = []
+    quarantine_ids: set[int] = set()
+    drop_ids: set[int] = set()
+    reasons: dict[int, list[str]] = {}
+    log: list[str] = []
+
+    for rule, res in zip(rules, results, strict=True):
+        if not res.found:
+            if mode is Mode.STRICT:
+                raise ValidationFailure(
+                    f"Validation failed under strict policy: column {res.column!r} "
+                    f"for check {res.check!r} is missing from the frame.",
+                    failures=[res],
+                )
+            log.append(f"validation skipped: column {res.column!r} not found")
+            warnings.warn(
+                f"CleanFrame: validation {res.check!r} skipped — column {res.column!r} is "
+                "not in the data. Use mode='strict' to fail instead.",
+                CleanFrameWarning,
+                stacklevel=2,
+            )
+            continue
+        if res.passed:
+            continue
+        action = _effective_policy(rule.on_fail, mode)
+        label = f"{res.column}:{res.check}"
+        if action == "error":
+            error_failures.append(res)
+        elif action == "warn":
+            log.append(f"warn: {res.n_failed} row(s) failed {label}")
+        elif action == "null":
+            null_actions.append((res.column, res.failed_row_ids))
+            log.append(f"null: blanked {res.n_failed} cell(s) failing {label}")
+        elif action == "drop":
+            drop_ids.update(res.failed_row_ids)
+            for rid in res.failed_row_ids:
+                reasons.setdefault(rid, []).append(label)
+            log.append(f"drop: removed {res.n_failed} row(s) failing {label}")
+        else:  # quarantine (default)
+            quarantine_ids.update(res.failed_row_ids)
+            for rid in res.failed_row_ids:
+                reasons.setdefault(rid, []).append(label)
+            log.append(f"quarantine: held {res.n_failed} row(s) failing {label}")
+
+    if error_failures:
+        detail = "; ".join(f"{r.column}:{r.check} ({r.n_failed} failed)" for r in error_failures)
+        raise ValidationFailure(
+            f"Validation failed under {mode.value} policy: {detail}", failures=error_failures
+        )
+
+    # Snapshot quarantine BEFORE null-blanking so a row that fails both a null rule
+    # and a quarantine rule is preserved with its original (failing) value for the
+    # human to inspect — not blanked to NaN.
+    quarantine_df = pd.DataFrame()
+    if quarantine_ids:
+        ordered = sorted(quarantine_ids)
+        quarantine_df = work.loc[ordered].copy()
+        quarantine_df[_reason_column(work)] = ["; ".join(reasons[i]) for i in ordered]
+
+    for col, ids in null_actions:
+        work.loc[ids, col] = np.nan
+
+    removed = quarantine_ids | drop_ids
+    removed_rows = [(rid, "; ".join(reasons.get(rid, ["validation"]))) for rid in sorted(removed)]
+    if removed:
+        work = work.drop(index=sorted(removed))
+
+    return ValidationOutcome(
+        dataframe=work,
+        quarantine=quarantine_df,
+        results=results,
+        removed_rows=removed_rows,
+        log=log,
+    )
+
+
+__all__ = [
+    "ValidationResult",
+    "ValidationOutcome",
+    "validator",
+    "list_validators",
+    "check_is_known",
+    "evaluate",
+    "apply_validations",
+    "pass_mask",
+    "QUARANTINE_REASON_COL",
+    "VALIDATOR_REGISTRY",
+]