cleanframe-engine 0.3.0__py3-none-any.whl

cleanframe/schema.py

@@ -0,0 +1,259 @@
+"""Target schemas: the shape you want your data to end up in.
+
+A schema is optional. When you pass one to :func:`cleanframe.clean`, the planner
+maps messy source columns onto your canonical columns (by fuzzy name + type match,
+with confidence) and derives validation rules from your constraints. When you
+don't have one, :func:`infer_schema` proposes a starting point from a clean-ish
+frame that you then edit and commit.
+
+Schemas are plain YAML, deliberately close to the recipe format so the two read
+alike in a PR.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+import yaml
+
+from ._util import load_yaml, read_text, snake_case, write_text
+from .errors import SchemaError
+from .profile import COMMON_DATE_FORMATS, _name_hint, profile_dataframe
+
+#: Map a profiler semantic type to a schema logical dtype.
+_SEMANTIC_TO_DTYPE = {
+    "integer": "integer",
+    "float": "float",
+    "currency": "float",
+    "boolean": "boolean",
+    "date": "date",
+    "datetime": "datetime",
+    "email": "email",
+    "phone": "phone",
+    "url": "url",
+    "categorical": "category",
+    "id": "string",
+    "text": "string",
+    "empty": "string",
+}
+
+#: How many distinct values a category may have before we stop enumerating them.
+_MAX_ALLOWED_VALUES = 50
+
+#: The logical dtypes a schema column may declare. A typo ("flaot", "emial") would
+#: otherwise be silently ignored (no cast, no warning), so it is rejected at load.
+_VALID_DTYPES = frozenset(
+    {
+        "string", "str", "text", "integer", "int", "float", "number", "boolean",
+        "bool", "date", "datetime", "email", "phone", "url", "category", "id",
+    }
+)
+
+
+#: Spellings that mean the same logical type. Normalised at load so the planner,
+#: which matches on the canonical name, cannot silently skip a cast.
+_DTYPE_ALIASES = {
+    "int": "integer",
+    "number": "float",
+    "text": "string",
+    "bool": "boolean",
+    "id": "string",
+    "str": "string",
+}
+
+
+def _canonical_dtype(dtype: Any, column: str) -> str:
+    if not isinstance(dtype, str):
+        raise SchemaError(
+            f"Schema column {column!r} dtype must be a string, got {type(dtype).__name__}."
+        )
+    name = dtype.strip().lower()
+    if name not in _VALID_DTYPES:
+        raise SchemaError(
+            f"Schema column {column!r} has unknown dtype {dtype!r}. "
+            f"Valid dtypes: {', '.join(sorted(_VALID_DTYPES))}."
+        )
+    return _DTYPE_ALIASES.get(name, name)
+
+
+@dataclass
+class SchemaColumn:
+    """One column in a target schema: a name, a logical type, and constraints."""
+
+    name: str
+    dtype: str = "string"
+    required: bool = False
+    unique: bool = False
+    allowed_values: list[Any] | None = None
+    date_formats: list[str] | None = None
+    min: float | None = None
+    max: float | None = None
+    aliases: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        out: dict[str, Any] = {"dtype": self.dtype}
+        if self.required:
+            out["required"] = True
+        if self.unique:
+            out["unique"] = True
+        if self.allowed_values is not None:
+            out["allowed_values"] = list(self.allowed_values)
+        if self.date_formats:
+            out["date_formats"] = list(self.date_formats)
+        if self.min is not None:
+            out["min"] = self.min
+        if self.max is not None:
+            out["max"] = self.max
+        if self.aliases:
+            out["aliases"] = list(self.aliases)
+        return out
+
+    @classmethod
+    def from_dict(cls, name: str, raw: Any) -> SchemaColumn:
+        if raw is None:
+            return cls(name=name)
+        if isinstance(raw, str):  # shorthand: `col: float`
+            return cls(name=name, dtype=_canonical_dtype(raw, name))
+        if not isinstance(raw, dict):
+            raise SchemaError(f"Schema column {name!r} must be a mapping or a dtype string.")
+        aliases = raw.get("aliases", []) or []
+        if isinstance(aliases, str):
+            aliases = [aliases]
+        if not isinstance(aliases, (list, tuple)):
+            raise SchemaError(f"Schema column {name!r} aliases must be a list of names.")
+        return cls(
+            name=name,
+            dtype=_canonical_dtype(raw.get("dtype", "string"), name),
+            required=bool(raw.get("required", False)),
+            unique=bool(raw.get("unique", False)),
+            allowed_values=raw.get("allowed_values"),
+            date_formats=raw.get("date_formats"),
+            min=raw.get("min"),
+            max=raw.get("max"),
+            aliases=[str(a) for a in aliases],
+        )
+
+
+@dataclass
+class Schema:
+    """An ordered set of target columns."""
+
+    columns: list[SchemaColumn] = field(default_factory=list)
+    name: str | None = None
+
+    def column(self, name: str) -> SchemaColumn | None:
+        for c in self.columns:
+            if c.name == name:
+                return c
+        return None
+
+    @property
+    def column_names(self) -> list[str]:
+        return [c.name for c in self.columns]
+
+    # -- serialisation ---------------------------------------------------
+    def to_dict(self) -> dict:
+        out: dict[str, Any] = {"version": 1}
+        if self.name:
+            out["name"] = self.name
+        out["columns"] = {c.name: c.to_dict() for c in self.columns}
+        return out
+
+    def to_yaml(self) -> str:
+        return yaml.safe_dump(
+            self.to_dict(), sort_keys=False, allow_unicode=True, default_flow_style=False, width=100
+        )
+
+    def save(self, path: str | Path) -> Path:
+        return write_text(path, self.to_yaml())
+
+    @classmethod
+    def from_dict(cls, raw: dict) -> Schema:
+        if not isinstance(raw, dict):
+            raise SchemaError(
+                f"A schema must be a mapping at the top level, got {type(raw).__name__}."
+            )
+        raw_cols = raw.get("columns", raw if "columns" not in raw and "version" not in raw else {})
+        if not isinstance(raw_cols, dict):
+            raise SchemaError("Schema 'columns' must be a mapping of name -> spec.")
+        columns = [SchemaColumn.from_dict(str(n), spec) for n, spec in raw_cols.items()]
+        return cls(columns=columns, name=raw.get("name"))
+
+    @classmethod
+    def load(cls, path: str | Path) -> Schema:
+        path = Path(path)
+        if not path.exists():
+            raise SchemaError(f"Schema not found: {path}")
+        if path.is_dir():
+            raise SchemaError(f"Schema path is a directory, not a file: {path}")
+        return cls.from_dict(load_yaml(read_text(path), error=SchemaError, what="schema"))
+
+
+def _infer_date_formats(series: pd.Series) -> list[str]:
+    """Which of the common date formats actually parse this column (order preserved)."""
+    values = series.dropna().astype(str)
+    if values.empty:
+        return []
+    found: list[str] = []
+    remaining = values
+    for fmt in COMMON_DATE_FORMATS:
+        if remaining.empty:
+            break
+        parsed = pd.to_datetime(remaining, format=fmt, errors="coerce")
+        if parsed.notna().any():
+            found.append(fmt)
+            remaining = remaining[parsed.isna()]
+    return found
+
+
+def infer_schema(df: pd.DataFrame, name: str | None = None) -> Schema:
+    """Propose a target :class:`Schema` from a (reasonably clean) dataframe.
+
+    A convenience for bootstrapping: profile the frame, translate each column's
+    semantic type into a logical dtype, and capture obvious constraints (no-nulls
+    → ``required``, all-unique → ``unique``, low-cardinality → ``allowed_values``,
+    numeric range → ``min``/``max``). Treat the result as a first draft to edit,
+    not gospel.
+    """
+    profile = profile_dataframe(df)
+    columns: list[SchemaColumn] = []
+    for cp in profile.columns:
+        dtype = _SEMANTIC_TO_DTYPE.get(cp.semantic_type, "string")
+        # A target schema uses canonical (snake_case) names so that it agrees with
+        # the planner's default output names; the original messy name becomes an
+        # alias so column mapping still recognises it. Without this, an inferred
+        # schema's names would not match the cleaned output and its validations
+        # would silently apply to nothing.
+        canonical = snake_case(cp.name) or cp.name
+        aliases = [cp.name] if canonical != cp.name else []
+        # Only infer `unique` for key-like columns. A tiny sample makes *everything*
+        # look unique; asserting it on, say, an amount column would wrongly reject
+        # valid future rows (and cleaning can even collapse distinct raw strings —
+        # "₹1,200" and "₹1200" — into one value).
+        key_like = cp.semantic_type in ("id", "email") or _name_hint(cp.name, "id")
+        col = SchemaColumn(
+            name=canonical,
+            dtype=dtype,
+            required=cp.null_count == 0 and cp.count > 0,
+            unique=cp.is_unique and key_like,
+            aliases=aliases,
+        )
+        if dtype == "category" and cp.count:
+            distinct = sorted({str(v) for v in df[cp.name].dropna().unique()})
+            if len(distinct) <= _MAX_ALLOWED_VALUES:
+                col.allowed_values = distinct
+        if dtype in ("date", "datetime"):
+            fmts = _infer_date_formats(df[cp.name])
+            if fmts:
+                col.date_formats = fmts
+        if dtype in ("integer", "float") and cp.numeric_stats:
+            col.min = cp.numeric_stats["min"]
+            col.max = cp.numeric_stats["max"]
+        columns.append(col)
+    return Schema(columns=columns, name=name)
+
+
+__all__ = ["Schema", "SchemaColumn", "infer_schema"]

cleanframe/streaming.py

@@ -0,0 +1,354 @@
+"""Out-of-core replay: stream a recipe over a file too large to fit in memory.
+
+Streaming applies ONLY to recipe *replay* (not planning — profiling/detection are
+whole-frame). It is correct only for **row-independent** recipes: every op and
+validation must produce the same result on a chunk as on the whole frame. That
+subset streams with byte-identical output; anything else is a **hard, named
+refusal**, never a silent divergence — because a misclassified global op would
+silently corrupt output, the exact failure the library exists to prevent.
+
+Refused (need the whole column / cross-row state): ``dedup``; ``fill_na`` with a
+mean/median/mode/ffill/bfill strategy; ``cast`` to ``category``/``datetime``/``date``;
+``parse_date`` *without* explicit formats (format inference is order-dependent);
+the ``unique`` validator; and any op/validator not on the streamable allow-list
+(default-DENY, so an unknown custom op is refused until proven row-independent).
+
+Row-ids in the (bounded) diff summary are global — offset by the chunk's start.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import warnings
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from ._util import check_output_target, sanitize_dataframe_for_csv
+from .errors import CleanFrameError, CleanFrameWarning
+from .executor import execute
+from .recipe import Recipe
+from .types import Mode
+
+# Column/frame ops that are provably row-independent (allow-list; default-DENY).
+_STREAMABLE_OPS = frozenset(
+    {
+        "strip_whitespace", "collapse_whitespace", "lowercase", "uppercase", "title_case",
+        "capitalize", "remove_symbols", "replace", "to_na", "normalize_email",
+        "normalize_phone", "parse_number", "round", "normalize_values", "extract_currency",
+        "normalize_unit", "drop_columns",
+    }
+)
+_CMP_RE = re.compile(r"^(>=|<=|==|!=|>|<)\s*-?\d+(?:\.\d+)?$")
+_CSV_STREAM_SUFFIXES = {".csv", ".txt", ".tsv", ""}
+
+
+def is_op_streamable(op) -> bool:
+    """True if applying ``op`` chunk-by-chunk equals applying it to the whole frame."""
+    if op.name == "cast":
+        return str(op.params.get("to", "")).lower() not in ("category", "datetime", "date")
+    if op.name == "parse_date":
+        return bool(op.params.get("formats"))  # inference over the series is order-dependent
+    if op.name == "fill_na":
+        return op.params.get("strategy") in (None, "zero", "empty")  # constant fill only
+    return op.name in _STREAMABLE_OPS
+
+
+def is_validation_streamable(rule) -> bool:
+    """True for row-local checks; ``unique`` (and unknown custom checks) are not."""
+    check = rule.check.strip()
+    if check == "unique":
+        return False
+    if check in ("not_null", "valid_email", "valid_url", "valid_phone"):
+        return True
+    if _CMP_RE.match(check):
+        return True
+    # Mirror validate.pass_mask membership / regex prefixes (not bare startswith("in")).
+    if check == "in" or check.startswith("in ") or check.startswith("in["):
+        return True
+    if check.startswith("matches") or check.startswith("regex"):
+        return True
+    return False  # unknown custom validator -> refuse (default-DENY)
+
+
+def check_streamable(recipe: Recipe) -> None:
+    """Raise :class:`~cleanframe.errors.CleanFrameError` naming the first op/validation
+    that cannot stream. Returns ``None`` if the whole recipe is streamable."""
+    for col in recipe.columns:
+        for op in col.ops:
+            if not is_op_streamable(op):
+                raise CleanFrameError(
+                    f"Recipe is not streamable: op {op.name!r} on column {col.source!r} needs the "
+                    "whole column (global state). Run it whole-frame with apply_recipe(), or split "
+                    "the recipe so the streaming part is row-independent."
+                )
+    for op in recipe.frame_ops:
+        if not is_op_streamable(op):
+            raise CleanFrameError(
+                f"Recipe is not streamable: frame op {op.name!r} needs all rows. "
+                "Run it whole-frame with apply_recipe()."
+            )
+    for rule in recipe.validations:
+        if not is_validation_streamable(rule):
+            raise CleanFrameError(
+                f"Recipe is not streamable: validation {rule.column}:{rule.check} needs all rows "
+                "(e.g. 'unique'). Run it whole-frame with apply_recipe()."
+            )
+
+
+def _stream_dtypes(recipe: Recipe):
+    """Column dtypes to pin for a stream, taken from the recipe's fingerprint.
+
+    Chunked reads must pin dtypes or a later chunk can infer a different type from
+    an earlier one. Pinning everything to ``str`` does that but renders numbers
+    differently from a whole-frame replay (``2`` vs ``2.0``), so the fingerprint's
+    recorded types are used when it has them.
+    """
+    fp = recipe.source_fingerprint if isinstance(recipe.source_fingerprint, dict) else {}
+    dtypes = fp.get("dtypes") if isinstance(fp.get("dtypes"), dict) else None
+    if not dtypes:
+        return str
+    pinned: dict[str, Any] = {}
+    for col, dtype in dtypes.items():
+        name = str(dtype).lower()
+        if name.startswith(("int", "uint")):
+            pinned[str(col)] = "Int64"
+        elif name.startswith("float"):
+            pinned[str(col)] = "float64"
+        elif name.startswith("bool"):
+            pinned[str(col)] = "boolean"
+        else:
+            pinned[str(col)] = str
+    return pinned
+
+
+def _stream_read_kwargs(recipe: Recipe, *, pin_dtype: bool) -> dict:
+    """Build pandas ``read_csv`` kwargs from ``recipe.read``, matching ``apply_recipe``."""
+    from .dataio import _pandas_skiprows, _text_kwargs
+
+    read = dict(recipe.read or {})
+    if read.get("sheet") is not None:
+        raise CleanFrameError(
+            "Streaming does not support Excel sheet selection (recipe.read.sheet). "
+            "Use apply_recipe() / cleanframe apply without --chunksize for workbooks."
+        )
+    kwargs: dict = {
+        "encoding": read.get("encoding", "utf-8-sig"),
+        "index_col": False,
+    }
+    if read.get("sep"):
+        kwargs["sep"] = read["sep"]
+    if read.get("columns") is not None:
+        kwargs["usecols"] = list(read["columns"])
+    skiprows = _pandas_skiprows(read.get("skiprows"), int(read.get("blank_lines") or 0))
+    if skiprows is not None:
+        kwargs["skiprows"] = skiprows
+    if read.get("nrows") is not None:
+        kwargs["nrows"] = read["nrows"]
+    if read.get("text"):
+        kwargs.update(_text_kwargs())
+    elif pin_dtype:
+        kwargs["dtype"] = _stream_dtypes(recipe)
+    return kwargs
+
+
+@dataclass
+class StreamSummary:
+    """Counts from a streamed replay (no full cell-level diff is kept)."""
+
+    out_path: Path
+    rows_in: int = 0
+    rows_out: int = 0
+    changed_cells: int = 0
+    rows_dropped: int = 0
+    rows_quarantined: int = 0
+    chunks: int = 0
+    quarantine_path: Path | None = None
+
+    def render(self) -> str:
+        base = (
+            f"Streamed {self.rows_in:,} → {self.rows_out:,} rows in {self.chunks} chunk(s); "
+            f"{self.changed_cells:,} cell(s) changed, {self.rows_dropped:,} dropped."
+        )
+        if self.rows_quarantined:
+            base += f" {self.rows_quarantined:,} quarantined → {self.quarantine_path}."
+        return base
+
+
+def stream_apply(
+    recipe: Recipe | str | Path | dict,
+    in_path: str | Path,
+    out_path: str | Path,
+    *,
+    chunksize: int = 100_000,
+    mode: Mode | str = Mode.REVIEW,
+    quarantine_path: str | Path | None = None,
+    check_drift: bool = True,
+    on_drift: str = "error",
+    overwrite: bool = False,
+) -> StreamSummary:
+    """Replay ``recipe`` over ``in_path`` (CSV) in chunks, writing cleaned CSV to
+    ``out_path`` without ever holding the whole file in memory.
+
+    Refuses (raises) if the recipe contains any non-row-independent op/validation, so
+    the streamed output is byte-identical to a whole-frame replay (values). Column
+    dtypes are pinned to string on read so representation can't drift across chunks.
+
+    ``recipe.read`` selection (``columns`` / ``sep`` / ``encoding`` / ``skiprows`` /
+    ``nrows``) is applied the same way as :func:`apply_recipe`. Output is written to a
+    temporary sibling file and atomically replaced onto ``out_path`` when the stream
+    completes successfully.
+
+    Like :func:`apply_recipe`, schema drift is checked first (on a bounded head sample)
+    and, by default, refuses (``on_drift="error"``) so a drifted file is never silently
+    streamed with a stale recipe.
+    """
+    if not isinstance(recipe, Recipe):
+        from .api import _resolve_recipe
+
+        recipe = _resolve_recipe(recipe)
+    check_streamable(recipe)
+
+    if isinstance(chunksize, bool) or not isinstance(chunksize, int) or chunksize < 1:
+        raise CleanFrameError(
+            f"chunksize must be a positive number of rows, got {chunksize!r}."
+        )
+    in_path = Path(in_path)
+    if not in_path.exists():
+        raise CleanFrameError(f"Input file not found: {in_path}")
+    if not in_path.is_file():
+        raise CleanFrameError(f"Input path is not a file (is it a directory?): {in_path}")
+    out_path = check_output_target(out_path, in_path, overwrite=overwrite)
+    suffix = in_path.suffix.lower()
+    if suffix not in _CSV_STREAM_SUFFIXES:
+        raise CleanFrameError(
+            f"Streaming (--chunksize) only supports CSV-family files, not {suffix or 'this file'}. "
+            "Use apply_recipe() for Excel/Parquet/JSON."
+        )
+    mode = Mode.coerce(mode)
+    if on_drift not in ("error", "warn", "ignore"):
+        raise CleanFrameError(
+            f"on_drift must be one of ['error', 'warn', 'ignore'], got {on_drift!r}."
+        )
+    head_kwargs = _stream_read_kwargs(recipe, pin_dtype=False)
+    read_kwargs = _stream_read_kwargs(recipe, pin_dtype=True)
+    if isinstance(read_kwargs.get("dtype"), dict):
+        try:
+            pd.read_csv(in_path, **{**read_kwargs, "nrows": 5})
+        except Exception:  # noqa: BLE001 - the file no longer matches the recorded types
+            read_kwargs["dtype"] = str
+            warnings.warn(
+                "CleanFrame: the file does not match the column types recorded in the "
+                "recipe; streaming every column as text instead.",
+                CleanFrameWarning,
+                stacklevel=2,
+            )
+
+    if check_drift and not recipe.source_fingerprint:
+        warnings.warn(
+            "CleanFrame: recipe has no source_fingerprint — drift check skipped. "
+            "Re-plan or stamp a fingerprint for production replay.",
+            CleanFrameWarning,
+            stacklevel=2,
+        )
+
+    # Drift check on a bounded head (read UN-pinned so dtypes match the recipe's
+    # fingerprint, which was built from the original inferred-dtype read).
+    if check_drift and recipe.source_fingerprint:
+        from .drift import detect_drift
+        from .errors import DriftError
+        from .fingerprint import DEFAULT_SAMPLE_ROWS
+
+        try:
+            head = pd.read_csv(in_path, nrows=DEFAULT_SAMPLE_ROWS, **head_kwargs)
+        except Exception as exc:  # noqa: BLE001 - IO boundary
+            raise CleanFrameError(
+                f"Could not read {in_path.name} to check for drift: {exc}."
+            ) from exc
+        drift = detect_drift(head, recipe, source=str(in_path))
+        if drift.has_drift and (on_drift == "error" or mode is Mode.STRICT):
+            raise DriftError(drift.render(), report=drift)
+        if drift.has_drift and on_drift == "warn":
+            warnings.warn("CleanFrame: " + drift.render(), CleanFrameWarning, stacklevel=2)
+
+    summary = StreamSummary(out_path=Path(out_path))
+    q_path = Path(quarantine_path) if quarantine_path else None
+    q_tmp = Path(str(q_path) + ".cf-tmp") if q_path is not None else None
+    q_written = False
+    first = True
+    tmp_out = Path(str(out_path) + ".cf-tmp")
+
+    try:
+        reader = pd.read_csv(in_path, chunksize=chunksize, **read_kwargs)
+    except Exception as exc:  # noqa: BLE001
+        raise CleanFrameError(f"Could not open {in_path.name} for streaming: {exc}.") from exc
+
+    try:
+        for chunk in reader:
+            # max_diff_changes=0: count every change but store none (streaming keeps counts,
+            # not a full in-memory lineage — the invariant-5 cap, applied globally). The
+            # per-chunk "diff truncated" warning is expected here, so silence just that one.
+            with warnings.catch_warnings():
+                warnings.filterwarnings("ignore", message="CleanFrame: diff detail truncated")
+                result = execute(recipe, chunk, mode=mode, max_diff_changes=0)
+            cleaned = sanitize_dataframe_for_csv(result.dataframe)
+            cleaned.to_csv(
+                tmp_out, mode="w" if first else "a", header=first, index=False,
+                encoding="utf-8", lineterminator="\n",
+            )
+            summary.rows_in += len(chunk)
+            summary.rows_out += len(cleaned)
+            summary.changed_cells += result.diff.changed_cells
+            summary.rows_dropped += len(result.diff.dropped_rows)
+            summary.chunks += 1
+            if result.has_quarantine:
+                if q_path is None:
+                    q_path = Path(out_path).with_suffix(".quarantine.csv")
+                    q_tmp = Path(str(q_path) + ".cf-tmp")
+                assert q_tmp is not None
+                q = sanitize_dataframe_for_csv(result.quarantine)
+                q.to_csv(
+                    q_tmp, mode="w" if not q_written else "a", header=not q_written,
+                    index=False, encoding="utf-8", lineterminator="\n",
+                )
+                q_written = True
+                summary.rows_quarantined += len(result.quarantine)
+            first = False
+
+        if first:
+            # Empty input — emit a header-only CSV using the recipe's column projection
+            # (or the file's columns) so the output is a valid empty table, not headerless.
+            cols = list(read_kwargs.get("usecols") or [])
+            if not cols:
+                try:
+                    cols = list(pd.read_csv(in_path, nrows=0, **head_kwargs).columns)
+                except Exception:  # noqa: BLE001
+                    cols = [c.source for c in recipe.columns]
+            pd.DataFrame(columns=cols).to_csv(
+                tmp_out, index=False, encoding="utf-8", lineterminator="\n"
+            )
+
+        os.replace(tmp_out, out_path)
+        if q_written and q_path is not None and q_tmp is not None:
+            os.replace(q_tmp, q_path)
+    except Exception:
+        if tmp_out.exists():
+            tmp_out.unlink(missing_ok=True)
+        if q_tmp is not None and q_tmp.exists():
+            q_tmp.unlink(missing_ok=True)
+        raise
+
+    summary.quarantine_path = q_path if q_written else None
+    return summary
+
+
+__all__ = [
+    "stream_apply",
+    "check_streamable",
+    "is_op_streamable",
+    "is_validation_streamable",
+    "StreamSummary",
+]