cleanframe-engine 0.3.0__py3-none-any.whl

cleanframe/detectors/base.py

@@ -0,0 +1,221 @@
+"""The detector plugin system.
+
+A **detector** inspects data and returns :class:`~cleanframe.issues.Issues`, each
+optionally carrying a :class:`~cleanframe.issues.Proposal` (the fix). Detectors are
+the extension point CleanFrame is built around — the community owns the long tail
+of messy-data weirdness by writing ~15-line detectors::
+
+    @cf.detector("iban")
+    def detect_iban(series):
+        issues = cf.Issues()
+        ...
+        return issues
+
+Signatures are flexible. A detector takes the data object for its scope
+(``series`` for column detectors, ``df`` for frame detectors) and, optionally, a
+second :class:`DetectorContext` argument when it needs the column name, the
+profile, or a target schema. The runner introspects the signature and passes what
+you ask for, so the minimal form above just works.
+"""
+
+from __future__ import annotations
+
+import inspect
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from typing import Any
+
+import pandas as pd
+
+from ..issues import Issue, Issues
+from ..profile import ColumnProfile, DataFrameProfile
+
+
+@dataclass
+class DetectorContext:
+    """Everything a detector might want beyond the raw data object.
+
+    Column detectors get ``column``/``series``/``column_profile`` set; frame
+    detectors get them as ``None``. ``schema`` is present only when the caller
+    passed a target schema. ``options`` carries user knobs (region for phones, a
+    seed alias map for categories, thresholds, …).
+    """
+
+    df: pd.DataFrame
+    profile: DataFrameProfile
+    column: str | None = None
+    series: pd.Series | None = None
+    column_profile: ColumnProfile | None = None
+    schema: Any | None = None
+    options: dict[str, Any] | None = None
+
+    def option(self, key: str, default: Any = None) -> Any:
+        return (self.options or {}).get(key, default)
+
+
+@dataclass
+class DetectorSpec:
+    name: str
+    func: Callable[..., Any]
+    scope: str  # "column" | "frame"
+    priority: int
+    requires_schema: bool
+    wants_ctx: bool
+    doc: str = ""
+
+
+DETECTOR_REGISTRY: dict[str, DetectorSpec] = {}
+
+
+def _wants_ctx(func: Callable) -> bool:
+    """Does this detector accept a second (context) argument?"""
+    try:
+        params = [
+            p
+            for p in inspect.signature(func).parameters.values()
+            if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD, p.KEYWORD_ONLY)
+        ]
+    except (TypeError, ValueError):  # pragma: no cover - builtins without signatures
+        return False
+    if any(p.name == "ctx" for p in params):
+        return True
+    return len(params) >= 2
+
+
+def detector(
+    name: str,
+    *,
+    scope: str = "column",
+    priority: int = 100,
+    requires_schema: bool = False,
+) -> Callable[[Callable], Callable]:
+    """Register a detector under ``name``.
+
+    Parameters
+    ----------
+    scope:
+        ``"column"`` (called once per column with its Series) or ``"frame"``
+        (called once with the whole DataFrame).
+    priority:
+        Lower runs earlier. Only affects the *order issues are reported*; the
+        planner orders the resulting ops canonically, so priority is about
+        presentation, not correctness.
+    requires_schema:
+        If true, the detector is skipped unless a target schema was supplied.
+    """
+
+    if scope not in ("column", "frame"):
+        raise ValueError(f"scope must be 'column' or 'frame', got {scope!r}")
+
+    def decorator(func: Callable) -> Callable:
+        if name in DETECTOR_REGISTRY:
+            raise ValueError(f"Detector {name!r} is already registered.")
+        DETECTOR_REGISTRY[name] = DetectorSpec(
+            name=name,
+            func=func,
+            scope=scope,
+            priority=priority,
+            requires_schema=requires_schema,
+            wants_ctx=_wants_ctx(func),
+            doc=func.__doc__ or "",
+        )
+        return func
+
+    return decorator
+
+
+def unregister_detector(name: str) -> None:
+    """Remove a detector (used by tests and for overriding a built-in)."""
+    DETECTOR_REGISTRY.pop(name, None)
+
+
+def list_detectors() -> list[str]:
+    return sorted(DETECTOR_REGISTRY)
+
+
+def _normalize_result(result: Any) -> Iterable[Issue]:
+    if result is None:
+        return []
+    if isinstance(result, Issues):
+        return list(result)
+    if isinstance(result, Issue):
+        return [result]
+    if isinstance(result, (list, tuple)):
+        return [r for r in result if isinstance(r, Issue)]
+    raise TypeError(
+        f"A detector must return Issues / Issue / list / None, got {type(result).__name__}."
+    )
+
+
+def _invoke(spec: DetectorSpec, data: Any, ctx: DetectorContext) -> Iterable[Issue]:
+    result = spec.func(data, ctx) if spec.wants_ctx else spec.func(data)
+    return _normalize_result(result)
+
+
+def run_detectors(
+    df: pd.DataFrame,
+    *,
+    profile: DataFrameProfile | None = None,
+    schema: Any | None = None,
+    options: dict[str, Any] | None = None,
+    only: Iterable[str] | None = None,
+) -> Issues:
+    """Run every applicable detector and return the aggregated, stamped issues.
+
+    Deterministic: detectors run in ``(priority, name)`` order and columns in frame
+    order. Each issue is stamped with its detector name (and column, for column
+    detectors) if the detector didn't set them.
+    """
+    from .._util import ensure_string_columns
+    from ..profile import profile_dataframe  # local import avoids a cycle at module load
+
+    df = ensure_string_columns(df)
+    profile = profile or profile_dataframe(df)
+    options = options or {}
+    only_set = set(only) if only is not None else None
+    issues = Issues()
+
+    specs = sorted(DETECTOR_REGISTRY.values(), key=lambda s: (s.priority, s.name))
+    for spec in specs:
+        if only_set is not None and spec.name not in only_set:
+            continue
+        if spec.requires_schema and schema is None:
+            continue
+
+        if spec.scope == "column":
+            for col in df.columns:
+                col_name = str(col)
+                ctx = DetectorContext(
+                    df=df,
+                    profile=profile,
+                    column=col_name,
+                    series=df[col],
+                    column_profile=profile.column(col_name),
+                    schema=schema,
+                    options=options,
+                )
+                for issue in _invoke(spec, df[col], ctx):
+                    issue.detector = issue.detector or spec.name
+                    if issue.column is None:
+                        issue.column = col_name
+                    issues.append(issue)
+        else:
+            ctx = DetectorContext(
+                df=df, profile=profile, schema=schema, options=options
+            )
+            for issue in _invoke(spec, df, ctx):
+                issue.detector = issue.detector or spec.name
+                issues.append(issue)
+
+    return issues
+
+
+__all__ = [
+    "DetectorContext",
+    "DetectorSpec",
+    "DETECTOR_REGISTRY",
+    "detector",
+    "unregister_detector",
+    "list_detectors",
+    "run_detectors",
+]

cleanframe/detectors/categories.py

@@ -0,0 +1,199 @@
+"""Category canonicalisation.
+
+Finds values that *mean the same thing* but are spelled differently and proposes a
+``normalize_values`` mapping to a single canonical spelling. Two mechanisms, both
+deterministic:
+
+* **Exact-after-normalisation** clusters (``"Bengaluru"``/``"bengaluru "``/
+  ``"BENGALURU"``) — high confidence.
+* **Fuzzy typo** clusters (``"Banglore"`` vs ``"Bangalore"``) via a similarity
+  threshold — lower confidence.
+
+Semantic abbreviations (``"BLR"`` → ``"Bangalore"``) are deliberately *out of
+scope* for rules — that's domain knowledge for a human or the LLM planner. The
+canonical spelling for each cluster is the most frequent original (ties broken
+alphabetically), so results never depend on row order.
+"""
+
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+from .._util import is_string_like, normalize_key, sample_non_null, similarity
+from ..issues import Issues
+from ..types import Op, Severity
+from .base import DetectorContext, detector
+
+#: Above this cardinality a column is treated as free text, not categories.
+_MAX_CATEGORY_CARDINALITY = 60
+_FUZZY_THRESHOLD = 0.86
+#: A fuzzy typo cluster only merges when the variant is this much rarer than the
+#: canonical it would fold into. This stops two *both-frequent* look-alikes
+#: (``insured``/``uninsured``, ``activated``/``deactivated``) from collapsing —
+#: a real typo is nearly always far rarer than the correct spelling.
+_FUZZY_MERGE_MAX_RATIO = 0.34
+_WS = re.compile(r"\s+")
+
+
+def _clean(value: str) -> str:
+    """Whitespace-normalise a value so category clustering sees post-trim spellings.
+
+    Categories run *after* whitespace ops in a recipe, so clustering on the cleaned
+    form keeps the ``normalize_values`` map keys matching the data at that point and
+    prevents a whitespace-messy spelling from ever being chosen as canonical.
+    """
+    return _WS.sub(" ", value).strip()
+
+
+def _casing_rank(s: str) -> int:
+    """Tie-break spellings: short CODE > Title > Mixed > lower > UPPER.
+
+    Short all-caps tokens (``CA``, ``NY``, ``INR``) are codes whose canonical form
+    *is* upper case; anything longer reads better title-cased.
+    """
+    if s.isupper() and len(s.strip()) <= 3:
+        return 0
+    if s.istitle():
+        return 1
+    if not s.isupper() and not s.islower():
+        return 2
+    if s.islower():
+        return 3
+    return 4
+
+
+def _canonical(spellings: dict[str, int]) -> str:
+    """Canonical spelling: most frequent, then nicest casing, then alphabetical.
+
+    All three keys are deterministic, so the choice never depends on row order.
+    """
+    return sorted(spellings.items(), key=lambda kv: (-kv[1], _casing_rank(kv[0]), kv[0]))[0][0]
+
+
+#: A negated spelling is a different category, however similar it looks.
+_NEGATION_PREFIXES = (
+    "un", "non", "in", "im", "ir", "il", "dis", "de", "re", "anti", "no", "not",
+)
+
+
+def _is_negation_pair(a: str, b: str) -> bool:
+    """True when one spelling is the other plus a negation/repetition prefix."""
+    x, y = normalize_key(a), normalize_key(b)
+    if x == y:
+        return False
+    longer, shorter = (x, y) if len(x) > len(y) else (y, x)
+    if not shorter or not longer.endswith(shorter):
+        return False
+    return longer[: -len(shorter)] in _NEGATION_PREFIXES
+
+
+def _cluster(counts: dict[str, int], seed_map: dict[str, str] | None) -> dict[str, str]:
+    """Return a mapping ``{variant: canonical}`` covering only values that change."""
+    # 1) group by aggressive normalisation key
+    groups: dict[str, dict[str, int]] = {}
+    for value, n in counts.items():
+        key = normalize_key(value)
+        if not key:
+            continue  # punctuation-only values ("-", "?") are not category variants
+        groups.setdefault(key, {})[value] = n
+
+    canon_of_group = {key: _canonical(spellings) for key, spellings in groups.items()}
+
+    # 2) fuzzy-merge whole groups whose canonical spellings are near-identical.
+    #    Deterministic: iterate group keys in sorted order, attach to the first
+    #    (largest, then alphabetically-first) existing cluster within threshold.
+    ordered = sorted(groups, key=lambda k: (-sum(groups[k].values()), k))
+    cluster_rep: dict[str, str] = {}
+    reps: list[str] = []
+    for key in ordered:
+        canon = canon_of_group[key]
+        key_count = sum(groups[key].values())
+        match = None
+        for rep in reps:
+            # Only fold this group into `rep` if it is BOTH near-identical in
+            # spelling AND rare relative to it — otherwise two distinct, comparably
+            # frequent categories would be silently merged (semantic inversion).
+            rep_count = sum(groups[rep].values())
+            if _is_negation_pair(canon, canon_of_group[rep]):
+                continue
+            if (
+                similarity(canon, canon_of_group[rep]) >= _FUZZY_THRESHOLD
+                and key_count <= rep_count * _FUZZY_MERGE_MAX_RATIO
+            ):
+                match = rep
+                break
+        if match is None:
+            reps.append(key)
+            cluster_rep[key] = key
+        else:
+            cluster_rep[key] = match
+
+    # 3) build variant -> canonical, honouring any seed overrides first.
+    mapping: dict[str, str] = {}
+    for key, spellings in groups.items():
+        rep_key = cluster_rep[key]
+        canonical = canon_of_group[rep_key]
+        for value in spellings:
+            target = (seed_map or {}).get(value, canonical)
+            if value != target:
+                mapping[value] = target
+    # Sort keys so the emitted recipe is byte-identical regardless of input row
+    # order (the cleaned data is order-independent either way).
+    return dict(sorted(mapping.items()))
+
+
+@detector("categories", priority=50)
+def detect_categories(series: pd.Series, ctx: DetectorContext) -> Issues:
+    """Cluster case/whitespace/typo variants of a category into one canonical value."""
+    issues = Issues()
+    cp = ctx.column_profile
+    if cp is None or cp.count == 0:
+        return issues
+    if not is_string_like(series):
+        return issues
+    # Only meaningful for low-cardinality columns.
+    if cp.unique_count > _MAX_CATEGORY_CARDINALITY or cp.semantic_type in (
+        "email", "phone", "url", "currency", "date", "datetime", "id",
+    ):
+        return issues
+
+    counts: dict[str, int] = {}
+    for v in sample_non_null(series):
+        if isinstance(v, str):
+            cleaned = _clean(v)
+            counts[cleaned] = counts.get(cleaned, 0) + 1
+    if len(counts) < 2:
+        return issues
+
+    seed_map = ctx.option("category_map", {}).get(ctx.column) if ctx.option("category_map") else None
+    mapping = _cluster(counts, seed_map)
+    if not mapping:
+        return issues
+
+    distinct_before = len(counts)
+    distinct_after = len({counts_key if counts_key not in mapping else mapping[counts_key]
+                          for counts_key in counts})
+    # Confidence: purely case/space merges are safe; fuzzy typo merges less so.
+    fuzzy = any(normalize_key(k) != normalize_key(v) for k, v in mapping.items())
+    confidence = 0.7 if fuzzy else 0.9
+
+    issues.add(
+        "category_variants",
+        f"{len(mapping)} value(s) are variants of other categories "
+        f"({distinct_before} → {distinct_after} distinct)",
+        severity=Severity.WARNING,
+        confidence=confidence,
+        evidence={
+            "mapping": mapping,
+            "distinct_before": distinct_before,
+            "distinct_after": distinct_after,
+            "fuzzy": fuzzy,
+        },
+        ops=[Op("normalize_values", {"map": mapping})],
+    )
+    return issues
+
+
+__all__ = ["detect_categories"]

cleanframe/detectors/contacts.py

@@ -0,0 +1,105 @@
+"""Email and phone detectors.
+
+Emails get the safe, universally-correct normalisation (trim + lowercase) plus an
+invalid-address count. Phones get best-effort separator/country-code normalisation
+driven by the ``phone_country_code`` option. Neither drops data — validation rules
+(added by the planner for these semantic types) decide what happens to values that
+still fail.
+"""
+
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+from .._util import sample_non_null
+from ..issues import Issues, _cap_examples
+from ..profile import EMAIL_RE, _name_hint
+from ..types import Op, Severity
+from .base import DetectorContext, detector
+
+
+@detector("emails", priority=45)
+def detect_emails(series: pd.Series, ctx: DetectorContext) -> Issues:
+    """Normalise emails (trim + lowercase) and flag invalid addresses."""
+    issues = Issues()
+    cp = ctx.column_profile
+    if cp is None or cp.count == 0:
+        return issues
+    if cp.semantic_type != "email" and not _name_hint(ctx.column or "", "email"):
+        return issues
+
+    values = [v for v in sample_non_null(series) if isinstance(v, str)]
+    if not values:
+        return issues
+
+    invalid = [v for v in values if not EMAIL_RE.match(v.strip().lower())]
+    needs_norm = [v for v in values if v != v.strip().lower()]
+
+    if needs_norm:
+        issues.add(
+            "email_normalization",
+            f"{len(needs_norm)} email(s) need trimming/lowercasing",
+            severity=Severity.INFO,
+            confidence=0.95,
+            evidence={"count": len(needs_norm), "examples": _cap_examples(needs_norm)},
+            ops=[Op("normalize_email")],
+        )
+    if invalid:
+        issues.add(
+            "invalid_emails",
+            f"{len(invalid)} value(s) are not valid email addresses",
+            severity=Severity.ERROR if len(invalid) / len(values) > 0.02 else Severity.WARNING,
+            confidence=1.0,
+            evidence={"count": len(invalid), "examples": _cap_examples(invalid)},
+        )
+    return issues
+
+
+@detector("phones", priority=45)
+def detect_phones(series: pd.Series, ctx: DetectorContext) -> Issues:
+    """Normalise phone-number formatting and flag implausible digit counts."""
+    issues = Issues()
+    cp = ctx.column_profile
+    if cp is None or cp.count == 0:
+        return issues
+    if cp.semantic_type != "phone" and not _name_hint(ctx.column or "", "phone"):
+        return issues
+
+    values = [v if isinstance(v, str) else str(v) for v in sample_non_null(series)]
+    if not values:
+        return issues
+
+    digit_counts = [len(re.sub(r"\D", "", v)) for v in values]
+    invalid = [v for v, d in zip(values, digit_counts, strict=False) if not (7 <= d <= 15)]
+    # "Needs normalisation" = contains separators or lacks a leading +.
+    messy = [v for v in values if re.search(r"[ ()\-.]", v) or not v.strip().startswith("+")]
+
+    country_code = ctx.option("phone_country_code") or ctx.option("region")
+    if messy:
+        params = {"default_country_code": country_code} if country_code else {}
+        issues.add(
+            "phone_normalization",
+            f"{len(messy)} phone number(s) have inconsistent formatting",
+            severity=Severity.INFO,
+            confidence=0.8,
+            evidence={
+                "count": len(messy),
+                "examples": _cap_examples(messy),
+                "country_code": country_code,
+            },
+            ops=[Op("normalize_phone", params)],
+        )
+    if invalid:
+        issues.add(
+            "invalid_phones",
+            f"{len(invalid)} value(s) have an implausible number of digits",
+            severity=Severity.WARNING,
+            confidence=0.9,
+            evidence={"count": len(invalid), "examples": _cap_examples(invalid)},
+        )
+    return issues
+
+
+__all__ = ["detect_emails", "detect_phones"]

cleanframe/detectors/currency.py

@@ -0,0 +1,112 @@
+"""Currency / money-column detection.
+
+Turns ``₹1,20,000`` / ``$1,200`` / ``1200 INR`` into a typed float. When a column
+holds a single currency, the code is folded into the column name (``amount`` →
+``amount_inr``, matching the README). When a column *mixes* currencies — where the
+amount alone would be meaningless — it additionally splits out a ``*_currency``
+column so no information is lost.
+"""
+
+from __future__ import annotations
+
+import re
+
+import numpy as np
+import pandas as pd
+
+from .._util import sample_non_null, snake_case, token_set
+from ..issues import Issues, _cap_examples
+from ..ops import _detect_currency_scalar, _parse_number_scalar
+from ..types import Op, Severity
+from .base import DetectorContext, detector
+
+_EU_GROUPED_RE = re.compile(r"\d{1,3}(?:\.\d{3})+,\d+")
+_US_GROUPED_RE = re.compile(r"\d{1,3}(?:,\d{3})+")
+_COMMA_DECIMAL_RE = re.compile(r"\d,\d{1,2}(?!\d)")
+
+
+def _decimal_convention(values: list[str]) -> dict[str, str]:
+    """Infer European ``1.234,56`` grouping. Empty dict means the pandas default.
+
+    Parsing ``€1.200,50`` with the default convention yields 1.2005 while reporting
+    nothing unparseable, so the convention has to be decided from the values.
+    """
+    eu_grouped = sum(1 for v in values if _EU_GROUPED_RE.search(v))
+    us_grouped = sum(1 for v in values if _US_GROUPED_RE.search(v))
+    if us_grouped:
+        return {}
+    if eu_grouped:
+        return {"decimal": ",", "thousands": "."}
+    comma_decimal = sum(1 for v in values if _COMMA_DECIMAL_RE.search(v))
+    if comma_decimal and not any("." in v for v in values):
+        return {"decimal": ",", "thousands": "."}
+    return {}
+
+
+@detector("currency", priority=45)
+def detect_currency(series: pd.Series, ctx: DetectorContext) -> Issues:
+    """Detect money-as-text columns and propose parsing to a float (+ currency split)."""
+    issues = Issues()
+    cp = ctx.column_profile
+    if cp is None or cp.count == 0 or cp.semantic_type != "currency":
+        return issues
+
+    values = [v if isinstance(v, str) else str(v) for v in sample_non_null(series)]
+    codes = {c for c in (_detect_currency_scalar(v, None) for v in values) if isinstance(c, str)}
+
+    convention = _decimal_convention(values)
+    decimal = convention.get("decimal", ".")
+    thousands = convention.get("thousands", ",")
+    # How many non-null values fail to become a number? (report, don't hide.)
+    unparsed = sum(
+        1 for v in values if np.isnan(_parse_number_scalar(v, decimal, thousands, []))
+    )
+
+    snake = snake_case(ctx.column or series.name or "amount")
+    ops: list[Op] = []
+    rename_to: str | None = None
+
+    if len(codes) == 1:
+        code = next(iter(codes))
+        # Fold the currency into the name unless it's already there.
+        if code.lower() not in token_set(snake):
+            rename_to = f"{snake}_{code.lower()}"
+        else:
+            rename_to = snake
+        ops = [Op("parse_number", dict(convention))]
+        currency_note = f"single currency {code}"
+    else:
+        rename_to = snake
+        target = f"{snake}_currency"
+        ops = [Op("extract_currency", {"to": target}), Op("parse_number", dict(convention))]
+        currency_note = (
+            f"mixed currencies {sorted(codes)} — splitting out `{target}`"
+            if codes
+            else "no explicit currency code"
+        )
+
+    if rename_to == (ctx.column or series.name):
+        rename_to = None
+
+    sev = Severity.WARNING if unparsed else Severity.INFO
+    evidence = {
+        "currencies": sorted(codes),
+        "unparsed": unparsed,
+        "examples": _cap_examples(values),
+    }
+    if convention:
+        evidence["decimal_convention"] = "european"
+    issues.add(
+        "currency_format",
+        f"Money column stored as text ({currency_note})"
+        + (f"; {unparsed} value(s) unparseable" if unparsed else ""),
+        severity=sev,
+        confidence=0.95,
+        evidence=evidence,
+        ops=ops,
+        rename_to=rename_to,
+    )
+    return issues
+
+
+__all__ = ["detect_currency"]