edaprep 0.1.0__py3-none-any.whl

edaprep/__init__.py

@@ -0,0 +1,188 @@
+"""edaprep: transparent, leakage-safe EDA and ML preprocessing.
+
+Quick start
+-----------
+
+Understand a dataset::
+
+    import edaprep
+
+    profile = edaprep.profile(df, target="churn")
+    print(profile.summary())
+
+    report = edaprep.EDA(df, target="churn").analyze()
+    print(report.summary())
+
+Prepare it, automatically but not opaquely::
+
+    pipe = edaprep.AutoPipeline(target="churn", model_family="tree", random_state=42)
+    pipe.fit(train_df)
+    pipe.explain()                    # why every column was treated as it was
+
+    X_train = pipe.transform(train_df)
+    X_test  = pipe.transform(test_df)  # same fitted statistics, no leakage
+
+    print(pipe.report_.summary())
+
+Or say exactly what should happen::
+
+    pipe = (
+        edaprep.Pipeline(target="churn")
+        .handle_missing()
+        .handle_outliers(strategy="clip")
+        .encode_categorical()
+        .scale_numeric()
+    )
+    X = pipe.fit_transform(train_df)
+
+Design guarantees
+-----------------
+* Every learned statistic is fitted on the training frame only; ``transform`` is a
+  pure function of that fitted state.
+* Every automatic decision is inspectable (``pipe.plan_``), explainable
+  (``pipe.explain()``), overridable (``config.column("age").imputation = "mean"``)
+  and reproducible (``random_state``, serialisable plan and report).
+* Nothing is silently discarded.  Dropped columns, imputed values, grouped
+  categories and clipped rows are all counted and reported.
+"""
+
+from ._version import __version__
+from .config import AUTO, ColumnConfig, Config, Thresholds
+from .core.base import Transformer
+from .core.context import FitContext
+from .core.pipeline import AutoPipeline, Pipeline
+from .eda.analyzer import EDA, EDAReport
+from .exceptions import (
+    ConfigurationError,
+    DataError,
+    EdaPrepError,
+    EmptyDataError,
+    LeakageError,
+    NotFittedError,
+    SchemaError,
+    TransformationError,
+)
+from .planning.decisions import Decision, Plan, PlannedStep
+from .planning.planner import Planner
+from .planning.rules import Rule, RuleSet, default_rules
+from .preprocessing import (
+    CategoricalEncoder,
+    ColumnDropper,
+    ConstantFilter,
+    CorrelationFilter,
+    DataTypeInference,
+    DateTimeExpander,
+    DistributionTransformer,
+    DuplicateColumnFilter,
+    DuplicateRowHandler,
+    FrequencyEncoder,
+    MissingIndicator,
+    MissingnessFilter,
+    MissingValueHandler,
+    OneHotEncoder,
+    OrdinalEncoder,
+    OutlierHandler,
+    RareCategoryGrouper,
+    Scaler,
+    TargetEncoder,
+    TextColumnHandler,
+    VarianceFilter,
+    detect_outliers,
+)
+from .profiling.profiler import ColumnProfile, DatasetProfile, profile
+from .reporting.report import Report
+from .types import AnalysisLevel, ModelFamily, SemanticType, Severity, Stage
+
+__all__ = [
+    "__version__",
+    # entry points
+    "profile",
+    "EDA",
+    "AutoPipeline",
+    "Pipeline",
+    # configuration
+    "Config",
+    "ColumnConfig",
+    "Thresholds",
+    "AUTO",
+    # planning
+    "Planner",
+    "Plan",
+    "PlannedStep",
+    "Decision",
+    "Rule",
+    "RuleSet",
+    "default_rules",
+    # results
+    "DatasetProfile",
+    "ColumnProfile",
+    "Report",
+    "EDAReport",
+    # types
+    "SemanticType",
+    "ModelFamily",
+    "Stage",
+    "Severity",
+    "AnalysisLevel",
+    # extension
+    "Transformer",
+    "FitContext",
+    # transformers
+    "DataTypeInference",
+    "DateTimeExpander",
+    "DuplicateRowHandler",
+    "MissingValueHandler",
+    "MissingIndicator",
+    "OutlierHandler",
+    "detect_outliers",
+    "CategoricalEncoder",
+    "OneHotEncoder",
+    "OrdinalEncoder",
+    "FrequencyEncoder",
+    "TargetEncoder",
+    "RareCategoryGrouper",
+    "Scaler",
+    "DistributionTransformer",
+    "TextColumnHandler",
+    "ColumnDropper",
+    "ConstantFilter",
+    "MissingnessFilter",
+    "DuplicateColumnFilter",
+    "CorrelationFilter",
+    "VarianceFilter",
+    # exceptions
+    "EdaPrepError",
+    "ConfigurationError",
+    "NotFittedError",
+    "SchemaError",
+    "DataError",
+    "EmptyDataError",
+    "TransformationError",
+    "LeakageError",
+]
+
+
+def __getattr__(name: str):
+    """Lazily expose optional subpackages.
+
+    ``edaprep.visualization`` needs matplotlib, which is an optional dependency.
+    Importing it eagerly would make ``import edaprep`` fail for users who installed
+    only the core, so it is resolved on first access with a message that says what to
+    install.
+    """
+    if name == "visualization":
+        try:
+            # importlib rather than `from . import visualization`: the latter goes
+            # through _handle_fromlist, which calls getattr on this module again and
+            # re-enters __getattr__, recursing until the stack blows instead of
+            # surfacing the ImportError.
+            import importlib
+
+            module = importlib.import_module("edaprep.visualization")
+        except ImportError as exc:
+            raise ImportError(
+                "edaprep.visualization requires matplotlib, which is an optional "
+                "dependency. Install it with: pip install 'edaprep[visualization]'"
+            ) from exc
+        return module
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

edaprep/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

edaprep/backends/__init__.py

@@ -0,0 +1,11 @@
+"""Execution backends.
+
+A narrow protocol between the library and a dataframe implementation.  See
+:mod:`edaprep.backends.base` for what it covers and, more importantly, what it
+deliberately does not.
+"""
+
+from .base import Backend, get_backend, register_backend
+from .pandas_backend import PandasBackend
+
+__all__ = ["Backend", "PandasBackend", "get_backend", "register_backend"]

edaprep/backends/base.py

@@ -0,0 +1,167 @@
+"""The backend protocol.
+
+A narrow seam between "what the library wants done to a frame" and "how a particular
+dataframe implementation does it".  ``pandas_backend.PandasBackend`` is the only
+implementation today.
+
+Why it is this small
+--------------------
+This is not a general dataframe abstraction, and it is deliberately not trying to be.
+Wrapping every pandas call would be abstraction for its own sake: it would slow the
+common path, obscure the code, and buy nothing until a second backend exists.
+
+What it does cover is the handful of operations that are (a) hot enough to be worth
+routing through a seam and (b) the ones an Arrow or Polars implementation would
+genuinely do differently.  Everything else in the library calls pandas directly.
+
+The motivation is concrete rather than speculative: tabular ML frames routinely reach
+500,000 x 400, and an Arrow-backed implementation is a foreseeable need for frames of
+that shape and larger.  Nothing here is used to support a second backend yet.
+
+Before writing one
+------------------
+Read ``docs/performance.md`` first.  The one place in this library where a hand-written
+replacement for pandas looked obviously worthwhile turned out to be 2.1x *slower* than
+the pandas code it replaced, and was deleted.  Measure before committing to an
+implementation.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
+
+import numpy as np
+
+__all__ = ["Backend", "get_backend", "register_backend"]
+
+
+class Backend(ABC):
+    """Operations the library routes through a backend.
+
+    Implementations receive and return their own native frame and column types; the
+    library treats them as opaque except through these methods.
+    """
+
+    #: Short name, used by :func:`get_backend`.
+    name: str = "base"
+
+    # -- introspection ---------------------------------------------------------------
+
+    @abstractmethod
+    def is_frame(self, obj: Any) -> bool:
+        """True when ``obj`` is a frame this backend understands."""
+
+    @abstractmethod
+    def shape(self, frame: Any) -> Tuple[int, int]:
+        """``(n_rows, n_columns)``."""
+
+    @abstractmethod
+    def column_names(self, frame: Any) -> List[str]:
+        """Column names, in order."""
+
+    @abstractmethod
+    def dtype_of(self, frame: Any, column: str) -> str:
+        """A string naming the column's storage type."""
+
+    @abstractmethod
+    def memory_usage(self, frame: Any, deep: bool = True) -> Dict[str, int]:
+        """Per-column bytes, plus a ``"total"`` key."""
+
+    # -- column access ----------------------------------------------------------------
+
+    @abstractmethod
+    def get_column(self, frame: Any, column: str) -> Any:
+        """One column, in the backend's native representation."""
+
+    @abstractmethod
+    def to_float_array(self, frame: Any, column: str) -> np.ndarray:
+        """One column as a float64 NumPy array, with missing values as NaN.
+
+        Every numeric kernel in the library ultimately works on NumPy, so this is the
+        single conversion point a backend must provide.
+        """
+
+    @abstractmethod
+    def select(self, frame: Any, columns: Sequence[str]) -> Any:
+        """A frame restricted to ``columns``, without copying where possible."""
+
+    @abstractmethod
+    def assign(self, frame: Any, columns: Mapping[str, Any]) -> Any:
+        """A new frame with ``columns`` replaced or appended.
+
+        Must not mutate ``frame``.  Implementations should pass untouched columns
+        through by reference rather than copying the whole frame -- the copy discipline
+        described in ``docs/architecture.md`` section 6.
+        """
+
+    # -- aggregation -------------------------------------------------------------------
+
+    @abstractmethod
+    def null_mask(self, frame: Any, column: str) -> np.ndarray:
+        """Boolean array: True where the value is missing."""
+
+    @abstractmethod
+    def n_unique(self, frame: Any, column: str, dropna: bool = True) -> int:
+        """Distinct value count."""
+
+    @abstractmethod
+    def value_counts(
+        self, frame: Any, column: str, dropna: bool = True
+    ) -> List[Tuple[Any, int]]:
+        """``(value, count)`` pairs, most frequent first."""
+
+    @abstractmethod
+    def quantiles(
+        self, frame: Any, columns: Sequence[str], levels: Sequence[float]
+    ) -> np.ndarray:
+        """Quantile matrix of shape ``(len(levels), len(columns))``."""
+
+    @abstractmethod
+    def group_mean(
+        self, frame: Any, value_column: str, group_column: str
+    ) -> Dict[Any, float]:
+        """Mean of ``value_column`` per level of ``group_column``."""
+
+    @abstractmethod
+    def duplicated_rows(
+        self, frame: Any, subset: Optional[Sequence[str]] = None
+    ) -> np.ndarray:
+        """Boolean array: True where the row repeats an earlier one."""
+
+    # -- construction --------------------------------------------------------------------
+
+    @abstractmethod
+    def concat_columns(self, frames: Iterable[Any]) -> Any:
+        """Join frames side by side.  Indexes are assumed aligned."""
+
+    @abstractmethod
+    def take_rows(self, frame: Any, mask: np.ndarray) -> Any:
+        """The rows where ``mask`` is True."""
+
+    def __repr__(self) -> str:  # pragma: no cover - trivial
+        return f"{type(self).__name__}(name={self.name!r})"
+
+
+_REGISTRY: Dict[str, Backend] = {}
+
+
+def register_backend(backend: Backend) -> Backend:
+    """Make a backend available to :func:`get_backend`."""
+    _REGISTRY[backend.name] = backend
+    return backend
+
+
+def get_backend(name: str = "pandas") -> Backend:
+    """Look up a registered backend by name."""
+    if name not in _REGISTRY:
+        if name == "pandas":
+            from .pandas_backend import PandasBackend
+
+            return register_backend(PandasBackend())
+        available = ", ".join(sorted(_REGISTRY)) or "pandas"
+        raise ValueError(
+            f"No backend named {name!r} is registered. Available: {available}. "
+            f"Register one with edaprep.backends.register_backend(MyBackend())."
+        )
+    return _REGISTRY[name]

edaprep/backends/pandas_backend.py

@@ -0,0 +1,128 @@
+"""The pandas backend: the only implementation, and the reference one.
+
+Every method here is a thin adapter over a pandas call. That is the point: the protocol
+exists so a *different* implementation can be written, not to add a layer over the one
+that already works.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
+
+import numpy as np
+import pandas as pd
+
+from .base import Backend
+
+__all__ = ["PandasBackend"]
+
+
+class PandasBackend(Backend):
+    """Backend over ``pandas.DataFrame``."""
+
+    name = "pandas"
+
+    # -- introspection ---------------------------------------------------------------
+
+    def is_frame(self, obj: Any) -> bool:
+        return isinstance(obj, pd.DataFrame)
+
+    def shape(self, frame: pd.DataFrame) -> Tuple[int, int]:
+        return frame.shape
+
+    def column_names(self, frame: pd.DataFrame) -> List[str]:
+        return [str(c) for c in frame.columns]
+
+    def dtype_of(self, frame: pd.DataFrame, column: str) -> str:
+        return str(frame[column].dtype)
+
+    def memory_usage(self, frame: pd.DataFrame, deep: bool = True) -> Dict[str, int]:
+        usage = frame.memory_usage(index=True, deep=deep)
+        return {"total": int(usage.sum()), **{str(k): int(v) for k, v in usage.items()}}
+
+    # -- column access ----------------------------------------------------------------
+
+    def get_column(self, frame: pd.DataFrame, column: str) -> pd.Series:
+        return frame[column]
+
+    def to_float_array(self, frame: pd.DataFrame, column: str) -> np.ndarray:
+        series = frame[column]
+        if isinstance(series.dtype, pd.CategoricalDtype):
+            series = series.astype("float64")
+        try:
+            return series.to_numpy(dtype=np.float64, na_value=np.nan, copy=False)
+        except (TypeError, ValueError):
+            return pd.to_numeric(series, errors="coerce").to_numpy(dtype=np.float64)
+
+    def select(self, frame: pd.DataFrame, columns: Sequence[str]) -> pd.DataFrame:
+        return frame[list(columns)]
+
+    def assign(self, frame: pd.DataFrame, columns: Mapping[str, Any]) -> pd.DataFrame:
+        if not columns:
+            return frame
+        # Rebuild from the existing column blocks with only the named columns replaced,
+        # rather than `frame.copy()` then assigning: a step that touches 5 of 400
+        # columns then allocates 5 columns, not 400.
+        data: Dict[str, Any] = {}
+        for name in frame.columns:
+            key = str(name)
+            data[key] = columns.get(key, frame[name])
+        for key, value in columns.items():
+            if key not in data:
+                data[key] = value
+        return pd.DataFrame(data, index=frame.index, copy=False)
+
+    # -- aggregation -------------------------------------------------------------------
+
+    def null_mask(self, frame: pd.DataFrame, column: str) -> np.ndarray:
+        return frame[column].isna().to_numpy()
+
+    def n_unique(self, frame: pd.DataFrame, column: str, dropna: bool = True) -> int:
+        try:
+            return int(frame[column].nunique(dropna=dropna))
+        except TypeError:
+            # Unhashable cell values (lists, dicts); the string view is slower but is
+            # the only thing that can be counted at all.
+            return int(frame[column].astype(str).nunique(dropna=dropna))
+
+    def value_counts(
+        self, frame: pd.DataFrame, column: str, dropna: bool = True
+    ) -> List[Tuple[Any, int]]:
+        counts = frame[column].value_counts(dropna=dropna)
+        return [(index, int(value)) for index, value in counts.items()]
+
+    def quantiles(
+        self, frame: pd.DataFrame, columns: Sequence[str], levels: Sequence[float]
+    ) -> np.ndarray:
+        return frame[list(columns)].quantile(list(levels)).to_numpy()
+
+    def group_mean(
+        self, frame: pd.DataFrame, value_column: str, group_column: str
+    ) -> Dict[Any, float]:
+        grouped = frame.groupby(group_column, observed=True)[value_column].mean()
+        return {index: float(value) for index, value in grouped.items()}
+
+    def duplicated_rows(
+        self, frame: pd.DataFrame, subset: Optional[Sequence[str]] = None
+    ) -> np.ndarray:
+        try:
+            return frame.duplicated(subset=list(subset) if subset else None).to_numpy()
+        except TypeError:
+            return (
+                frame.astype(str)
+                .duplicated(subset=list(subset) if subset else None)
+                .to_numpy()
+            )
+
+    # -- construction --------------------------------------------------------------------
+
+    def concat_columns(self, frames: Iterable[pd.DataFrame]) -> pd.DataFrame:
+        parts = list(frames)
+        if not parts:
+            return pd.DataFrame()
+        if len(parts) == 1:
+            return parts[0]
+        return pd.concat(parts, axis=1, copy=False)
+
+    def take_rows(self, frame: pd.DataFrame, mask: np.ndarray) -> pd.DataFrame:
+        return frame.loc[np.asarray(mask, dtype=bool)]