xplainable-preprocessing 0.1.0__py3-none-any.whl

xplainable_preprocessing/__init__.py

@@ -0,0 +1,20 @@
+from xplainable_preprocessing.schema import PipelineSpec, StepSpec
+from xplainable_preprocessing.compiler import compile_spec
+from xplainable_preprocessing.schema import validate_spec
+from xplainable_preprocessing.serialization import save_pipeline, load_pipeline
+from xplainable_preprocessing.pipeline import DataFramePipeline, DataFrameColumnTransformer
+from xplainable_preprocessing.registry import REGISTRY, register, generate_catalog
+
+__all__ = [
+    "PipelineSpec",
+    "StepSpec",
+    "compile_spec",
+    "validate_spec",
+    "save_pipeline",
+    "load_pipeline",
+    "DataFramePipeline",
+    "DataFrameColumnTransformer",
+    "REGISTRY",
+    "register",
+    "generate_catalog",
+]

xplainable_preprocessing/compiler.py

@@ -0,0 +1,68 @@
+"""Compile a PipelineSpec into a DataFramePipeline."""
+
+from __future__ import annotations
+
+import inspect
+
+from xplainable_preprocessing.pipeline import DataFrameColumnTransformer, DataFramePipeline
+from xplainable_preprocessing.registry import REGISTRY
+from xplainable_preprocessing.sandbox import compile_custom
+from xplainable_preprocessing.schema import PipelineSpec
+
+
+def _coerce_params(cls, params: dict) -> dict:
+    """Convert list params to tuples where the constructor expects them.
+
+    JSON has no tuple type, so specs always contain lists. sklearn transformers
+    like MinMaxScaler expect tuples for params like feature_range.
+    """
+    sig = inspect.signature(cls.__init__)
+    coerced = {}
+    for key, value in params.items():
+        if isinstance(value, list) and key in sig.parameters:
+            param = sig.parameters[key]
+            if param.default is not inspect.Parameter.empty and isinstance(param.default, tuple):
+                value = tuple(value)
+        coerced[key] = value
+    return coerced
+
+
+def compile_spec(spec: PipelineSpec) -> DataFramePipeline:
+    """Convert a PipelineSpec into a DataFramePipeline.
+
+    Parameters
+    ----------
+    spec : PipelineSpec
+        The pipeline specification to compile.
+
+    Returns
+    -------
+    DataFramePipeline
+        An unfitted pipeline ready for .fit() and .transform().
+
+    Raises
+    ------
+    ValueError
+        If a step type is not found in the registry and is not "custom".
+    """
+    steps = []
+
+    for step in spec.steps:
+        if step.type == "custom":
+            transformer = compile_custom(step.params)
+        else:
+            if step.type not in REGISTRY:
+                raise ValueError(
+                    f"Step '{step.id}': unknown type '{step.type}'. "
+                    f"Available: {sorted(REGISTRY.keys())}"
+                )
+            cls = REGISTRY[step.type]
+            params = _coerce_params(cls, step.params)
+            transformer = cls(**params)
+
+        if step.columns:
+            transformer = DataFrameColumnTransformer(transformer, step.columns)
+
+        steps.append((step.id, transformer))
+
+    return DataFramePipeline(steps)

xplainable_preprocessing/pipeline.py

@@ -0,0 +1,87 @@
+"""DataFrame-preserving pipeline and column transformer."""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+
+class DataFrameColumnTransformer(BaseEstimator, TransformerMixin):
+    """Applies a transformer to specific columns, passes the rest through."""
+
+    def __init__(self, transformer: TransformerMixin, columns: list[str]):
+        self.transformer = transformer
+        self.columns = columns
+
+    def fit(self, X: pd.DataFrame, y=None):
+        self.transformer.fit(X[self.columns], y)
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        Xt = X.copy()
+        result = self.transformer.transform(Xt[self.columns])
+        if isinstance(result, pd.DataFrame):
+            Xt = Xt.drop(columns=self.columns)
+            Xt = pd.concat([Xt, result], axis=1)
+        elif hasattr(result, 'shape') and len(result.shape) == 2 and result.shape[1] != len(self.columns):
+            # Column count changed (e.g., OneHotEncoder) — get proper names
+            Xt = Xt.drop(columns=self.columns)
+            if hasattr(self.transformer, 'get_feature_names_out'):
+                new_cols = list(self.transformer.get_feature_names_out())
+            else:
+                new_cols = [f"{self.columns[0]}_{i}" for i in range(result.shape[1])]
+            result_df = pd.DataFrame(result, columns=new_cols, index=Xt.index)
+            Xt = pd.concat([Xt, result_df], axis=1)
+        else:
+            # numpy array with same column count — replace in place
+            Xt[self.columns] = result
+        return Xt
+
+    def fit_transform(self, X: pd.DataFrame, y=None) -> pd.DataFrame:
+        self.fit(X, y)
+        return self.transform(X)
+
+
+class DataFramePipeline(BaseEstimator, TransformerMixin):
+    """Sequential pipeline that preserves DataFrame structure.
+
+    Unlike sklearn's Pipeline which converts DataFrames to numpy arrays,
+    this pipeline passes DataFrames between steps.
+
+    Parameters
+    ----------
+    steps : list of (name, transformer) tuples
+        The ordered sequence of transformers to apply.
+    """
+
+    def __init__(self, steps: list[tuple[str, TransformerMixin]]):
+        self.steps = steps
+
+    def fit(self, X: pd.DataFrame, y=None):
+        Xt = X.copy()
+        for name, transformer in self.steps:
+            Xt = transformer.fit_transform(Xt, y)
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        Xt = X.copy()
+        for name, transformer in self.steps:
+            Xt = transformer.transform(Xt)
+        return Xt
+
+    def fit_transform(self, X: pd.DataFrame, y=None) -> pd.DataFrame:
+        Xt = X.copy()
+        for name, transformer in self.steps:
+            Xt = transformer.fit_transform(Xt, y)
+        return Xt
+
+    def get_step(self, name: str) -> TransformerMixin:
+        """Get a step by name."""
+        for step_name, transformer in self.steps:
+            if step_name == name:
+                return transformer
+        raise KeyError(f"Step '{name}' not found")
+
+    @property
+    def step_names(self) -> list[str]:
+        return [name for name, _ in self.steps]

xplainable_preprocessing/preview.py

@@ -0,0 +1,137 @@
+"""Preview and delta computation for UI display."""
+
+from __future__ import annotations
+
+import math
+
+import pandas as pd
+
+from xplainable_preprocessing.pipeline import DataFramePipeline
+
+
+def _sanitize_records(records: list[dict]) -> list[dict]:
+    """Replace NaN/Inf with None for JSON-safe output."""
+    sanitized = []
+    for row in records:
+        sanitized.append({
+            k: None if isinstance(v, float) and (math.isnan(v) or math.isinf(v)) else v
+            for k, v in row.items()
+        })
+    return sanitized
+
+
+def infer_schema(df: pd.DataFrame) -> dict:
+    """Infer column schema from a DataFrame.
+
+    Returns
+    -------
+    dict
+        {"columns": [{"name": str, "dtype": str}, ...],
+         "sample": [dict] (first row, JSON-safe)}
+    """
+    return {
+        "columns": [
+            {"name": col, "dtype": str(df[col].dtype)}
+            for col in df.columns
+        ],
+        "sample": _sanitize_records(df.head(1).to_dict("records")) if len(df) > 0 else [],
+    }
+
+
+def compute_delta(before: pd.DataFrame, after: pd.DataFrame) -> dict:
+    """Compute the transformation delta between two DataFrames.
+
+    Returns
+    -------
+    dict with keys:
+        - dropped: list of column names removed
+        - added: list of column names added
+        - updated: list of column names whose values changed
+        - rows_before: int
+        - rows_after: int
+    """
+    before_cols = set(before.columns)
+    after_cols = set(after.columns)
+
+    dropped = sorted(before_cols - after_cols)
+    added = sorted(after_cols - before_cols)
+
+    common = before_cols & after_cols
+    updated = []
+    for col in sorted(common):
+        try:
+            if not before[col].equals(after[col]):
+                updated.append(col)
+        except Exception:
+            updated.append(col)
+
+    return {
+        "dropped": dropped,
+        "added": added,
+        "updated": updated,
+        "rows_before": len(before),
+        "rows_after": len(after),
+    }
+
+
+def compute_step_deltas(
+    pipeline: DataFramePipeline,
+    df: pd.DataFrame,
+) -> list[dict]:
+    """Compute per-step deltas for a fitted pipeline.
+
+    Parameters
+    ----------
+    pipeline : DataFramePipeline
+        A fitted pipeline.
+    df : pd.DataFrame
+        Sample data to transform.
+
+    Returns
+    -------
+    list of dict
+        One delta per step, each containing step_id, step_type, and the delta.
+    """
+    deltas = []
+    Xt = df.copy()
+
+    for name, transformer in pipeline.steps:
+        before = Xt.copy()
+        Xt = transformer.transform(Xt)
+        delta = compute_delta(before, Xt)
+        deltas.append({
+            "step_id": name,
+            "delta": delta,
+            "sample_after": _sanitize_records(Xt.head(5).to_dict("records")),
+        })
+
+    return deltas
+
+
+def compute_preview(
+    pipeline: DataFramePipeline,
+    df: pd.DataFrame,
+) -> dict:
+    """Compute a full preview of pipeline transformation.
+
+    Returns
+    -------
+    dict with keys:
+        - input_schema: schema of input data
+        - output_schema: schema of output data
+        - step_deltas: per-step deltas
+        - sample_before: first 5 rows of input
+        - sample_after: first 5 rows of output
+    """
+    input_schema = infer_schema(df)
+    result = pipeline.transform(df)
+    output_schema = infer_schema(result)
+    step_deltas = compute_step_deltas(pipeline, df)
+
+    return {
+        "input_schema": input_schema,
+        "output_schema": output_schema,
+        "step_deltas": step_deltas,
+        "sample_before": _sanitize_records(df.head(5).to_dict("records")),
+        "sample_after": _sanitize_records(result.head(5).to_dict("records")),
+    }

xplainable_preprocessing/registry.py

@@ -0,0 +1,108 @@
+"""Transformer registry mapping names to sklearn-compatible classes."""
+
+from __future__ import annotations
+
+import inspect
+from typing import Any
+
+from sklearn.impute import SimpleImputer
+from sklearn.preprocessing import (
+    Binarizer,
+    KBinsDiscretizer,
+    MinMaxScaler,
+    OneHotEncoder,
+    OrdinalEncoder,
+    PowerTransformer,
+    QuantileTransformer,
+    RobustScaler,
+    StandardScaler,
+)
+
+from xplainable_preprocessing.transformers import (
+    CategoryCondenseTransformer,
+    DateTimeExtractTransformer,
+    DropColumnsTransformer,
+    ExpressionTransformer,
+    FillMissingTransformer,
+    GroupedLagTransformer,
+    RenameColumnsTransformer,
+    RollingAggTransformer,
+    TextCleanTransformer,
+    TypeCastTransformer,
+)
+
+# Registry: name -> class
+REGISTRY: dict[str, type] = {
+    # Standard sklearn
+    "SimpleImputer": SimpleImputer,
+    "StandardScaler": StandardScaler,
+    "MinMaxScaler": MinMaxScaler,
+    "RobustScaler": RobustScaler,
+    "OneHotEncoder": OneHotEncoder,
+    "OrdinalEncoder": OrdinalEncoder,
+    "PowerTransformer": PowerTransformer,
+    "QuantileTransformer": QuantileTransformer,
+    "KBinsDiscretizer": KBinsDiscretizer,
+    "Binarizer": Binarizer,
+    # Custom transformers
+    "ExpressionTransformer": ExpressionTransformer,
+    "DropColumnsTransformer": DropColumnsTransformer,
+    "RenameColumnsTransformer": RenameColumnsTransformer,
+    "TypeCastTransformer": TypeCastTransformer,
+    "FillMissingTransformer": FillMissingTransformer,
+    "CategoryCondenseTransformer": CategoryCondenseTransformer,
+    "TextCleanTransformer": TextCleanTransformer,
+    "DateTimeExtractTransformer": DateTimeExtractTransformer,
+    "GroupedLagTransformer": GroupedLagTransformer,
+    "RollingAggTransformer": RollingAggTransformer,
+}
+
+
+def generate_catalog() -> str:
+    """Generate a prompt-ready catalog of available transformer types.
+
+    Introspects the REGISTRY to produce a formatted string listing
+    each transformer with its constructor parameters, defaults, and
+    a one-line description from the class docstring.
+
+    Returns a string suitable for embedding in LLM system prompts.
+    """
+    lines = []
+    for name, cls in REGISTRY.items():
+        # First line of docstring as description
+        doc = (cls.__doc__ or "").strip().split("\n")[0]
+
+        # Constructor signature (skip 'self')
+        sig = inspect.signature(cls.__init__)
+        params = []
+        for pname, param in sig.parameters.items():
+            if pname == "self":
+                continue
+            if param.default is inspect.Parameter.empty:
+                params.append(pname)
+            else:
+                params.append(f"{pname}={param.default!r}")
+
+        param_str = ", ".join(params)
+        lines.append(f"- {name}({param_str}) — {doc}")
+
+    # Add custom code option
+    lines.append(
+        '- custom(code="...", class_name="...", description="...") '
+        "— Write a custom sklearn-compatible transformer when no built-in type fits"
+    )
+
+    return "\n".join(lines)
+
+
+def register(name: str, cls: type) -> None:
+    """Register a new transformer class.
+
+    Parameters
+    ----------
+    name : str
+        The name to register the transformer under.
+    cls : type
+        The transformer class (must be sklearn-compatible).
+    """
+    REGISTRY[name] = cls

xplainable_preprocessing/sandbox.py

@@ -0,0 +1,113 @@
+"""AST-based code validator and restricted executor for custom transformers."""
+
+from __future__ import annotations
+
+import ast
+
+FORBIDDEN_IMPORTS = {
+    "os", "sys", "subprocess", "socket", "shutil", "pathlib",
+    "http", "urllib", "requests", "importlib", "ctypes", "signal",
+    "multiprocessing", "threading", "asyncio", "pickle", "shelve",
+    "tempfile", "glob", "io", "builtins", "code", "codeop",
+}
+
+FORBIDDEN_CALLS = {
+    "exec", "eval", "__import__", "open", "compile",
+    "globals", "locals", "breakpoint", "exit", "quit",
+    "getattr", "setattr", "delattr",
+}
+
+
+def validate_code(code: str) -> None:
+    """AST-validate custom transformer code. Raises ValueError if unsafe."""
+    try:
+        tree = ast.parse(code)
+    except SyntaxError as e:
+        raise ValueError(f"Syntax error in custom code: {e}") from e
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                root_module = alias.name.split(".")[0]
+                if root_module in FORBIDDEN_IMPORTS:
+                    raise ValueError(f"Forbidden import: {alias.name}")
+
+        elif isinstance(node, ast.ImportFrom):
+            if node.module:
+                root_module = node.module.split(".")[0]
+                if root_module in FORBIDDEN_IMPORTS:
+                    raise ValueError(f"Forbidden import: {node.module}")
+
+        elif isinstance(node, ast.Call):
+            if isinstance(node.func, ast.Name):
+                if node.func.id in FORBIDDEN_CALLS:
+                    raise ValueError(f"Forbidden call: {node.func.id}")
+            elif isinstance(node.func, ast.Attribute):
+                if node.func.attr in FORBIDDEN_CALLS:
+                    raise ValueError(f"Forbidden call: {node.func.attr}")
+
+
+def compile_custom(params: dict):
+    """Compile custom transformer code into a class instance.
+
+    Validates the code via AST, executes it in a restricted namespace,
+    and returns an instance of the specified class.
+
+    Parameters
+    ----------
+    params : dict
+        Must contain 'code' (str) and 'class_name' (str).
+        May contain additional constructor kwargs.
+
+    Returns
+    -------
+    transformer : object
+        An instance of the custom transformer class.
+    """
+    code = params["code"]
+    class_name = params["class_name"]
+
+    validate_code(code)
+
+    # Execute in a restricted namespace with limited builtins
+    # Defense in depth: restrict __builtins__ to prevent class-hierarchy escapes
+    safe_builtins = {
+        name: __builtins__[name] if isinstance(__builtins__, dict) else getattr(__builtins__, name)
+        for name in (
+            "__build_class__",  # Required for class definitions
+            "True", "False", "None", "int", "float", "str", "bool", "list",
+            "dict", "tuple", "set", "frozenset", "range", "enumerate", "zip",
+            "map", "filter", "sorted", "reversed", "len", "min", "max", "sum",
+            "abs", "round", "isinstance", "issubclass", "type", "super",
+            "property", "staticmethod", "classmethod", "print", "repr",
+            "hasattr", "ValueError", "TypeError", "KeyError", "IndexError",
+            "AttributeError", "RuntimeError", "Exception", "NotImplementedError",
+            "StopIteration", "object", "slice", "any", "all",
+        )
+        if (isinstance(__builtins__, dict) and name in __builtins__)
+        or (not isinstance(__builtins__, dict) and hasattr(__builtins__, name))
+    }
+    # Allow __import__ for whitelisted modules only
+    safe_builtins["__import__"] = __builtins__["__import__"] if isinstance(__builtins__, dict) else __builtins__.__import__
+
+    namespace = {
+        "__builtins__": safe_builtins,
+        "__name__": "__custom_transformer__",
+    }
+    exec(code, namespace)  # noqa: S102 - AST-validated above
+
+    if class_name not in namespace:
+        raise ValueError(
+            f"Class '{class_name}' not found in custom code. "
+            f"Available: {[k for k in namespace if not k.startswith('_')]}"
+        )
+
+    cls = namespace[class_name]
+
+    # Extract constructor kwargs (exclude code/class_name/description)
+    constructor_params = {
+        k: v for k, v in params.items()
+        if k not in ("code", "class_name", "description")
+    }
+
+    return cls(**constructor_params)

xplainable_preprocessing/schema.py

@@ -0,0 +1,71 @@
+"""Pydantic models for pipeline specification."""
+
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, field_validator
+
+
+class StepSpec(BaseModel):
+    """A single preprocessing step in the pipeline."""
+
+    id: str
+    type: str
+    columns: Optional[List[str]] = None
+    params: Dict = {}
+    description: Optional[str] = None
+
+    @field_validator("id")
+    @classmethod
+    def id_must_be_non_empty(cls, v: str) -> str:
+        if not v.strip():
+            raise ValueError("Step id must be non-empty")
+        return v
+
+    @field_validator("type")
+    @classmethod
+    def type_must_be_non_empty(cls, v: str) -> str:
+        if not v.strip():
+            raise ValueError("Step type must be non-empty")
+        return v
+
+
+class PipelineSpec(BaseModel):
+    """Full pipeline specification containing ordered steps."""
+
+    version: str = "2.0"
+    steps: List[StepSpec] = []
+
+    @field_validator("steps")
+    @classmethod
+    def step_ids_must_be_unique(cls, v: List[StepSpec]) -> List[StepSpec]:
+        ids = [step.id for step in v]
+        if len(ids) != len(set(ids)):
+            duplicates = [id_ for id_ in ids if ids.count(id_) > 1]
+            raise ValueError(f"Duplicate step ids: {set(duplicates)}")
+        return v
+
+
+def validate_spec(spec: PipelineSpec) -> None:
+    """Validate a PipelineSpec. Raises ValueError if invalid.
+
+    Checks beyond Pydantic validation:
+    - All non-custom types must exist in the registry
+    - Custom steps must have 'code' and 'class_name' in params
+    """
+    from xplainable_preprocessing.registry import REGISTRY
+
+    for step in spec.steps:
+        if step.type == "custom":
+            if "code" not in step.params:
+                raise ValueError(
+                    f"Step '{step.id}': custom type requires 'code' in params"
+                )
+            if "class_name" not in step.params:
+                raise ValueError(
+                    f"Step '{step.id}': custom type requires 'class_name' in params"
+                )
+        elif step.type not in REGISTRY:
+            raise ValueError(
+                f"Step '{step.id}': unknown type '{step.type}'. "
+                f"Available: {sorted(REGISTRY.keys())}"
+            )

xplainable_preprocessing/serialization.py

@@ -0,0 +1,20 @@
+"""Cloudpickle-based pipeline serialization."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import cloudpickle
+
+if TYPE_CHECKING:
+    from xplainable_preprocessing.pipeline import DataFramePipeline
+
+
+def save_pipeline(pipeline: DataFramePipeline) -> bytes:
+    """Serialize a fitted pipeline to bytes."""
+    return cloudpickle.dumps(pipeline)
+
+
+def load_pipeline(data: bytes) -> DataFramePipeline:
+    """Deserialize a fitted pipeline from bytes."""
+    return cloudpickle.loads(data)  # noqa: S301 - trusted internal data

xplainable_preprocessing/transformers/__init__.py

@@ -0,0 +1,23 @@
+from xplainable_preprocessing.transformers.expression import ExpressionTransformer
+from xplainable_preprocessing.transformers.drop_columns import DropColumnsTransformer
+from xplainable_preprocessing.transformers.rename_columns import RenameColumnsTransformer
+from xplainable_preprocessing.transformers.type_cast import TypeCastTransformer
+from xplainable_preprocessing.transformers.fill_missing import FillMissingTransformer
+from xplainable_preprocessing.transformers.category_condense import CategoryCondenseTransformer
+from xplainable_preprocessing.transformers.text_clean import TextCleanTransformer
+from xplainable_preprocessing.transformers.datetime_extract import DateTimeExtractTransformer
+from xplainable_preprocessing.transformers.grouped_lag import GroupedLagTransformer
+from xplainable_preprocessing.transformers.rolling_agg import RollingAggTransformer
+
+__all__ = [
+    "ExpressionTransformer",
+    "DropColumnsTransformer",
+    "RenameColumnsTransformer",
+    "TypeCastTransformer",
+    "FillMissingTransformer",
+    "CategoryCondenseTransformer",
+    "TextCleanTransformer",
+    "DateTimeExtractTransformer",
+    "GroupedLagTransformer",
+    "RollingAggTransformer",
+]

xplainable_preprocessing/transformers/category_condense.py

@@ -0,0 +1,50 @@
+"""Category condensing transformer."""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+
+class CategoryCondenseTransformer(BaseEstimator, TransformerMixin):
+    """Collapse low-frequency categories into an 'Other' category.
+
+    Parameters
+    ----------
+    max_categories : int
+        Maximum number of categories to keep. The rest become `other_label`.
+    other_label : str
+        Label for condensed categories. Default "Other".
+    min_frequency : float | None
+        If set, categories with frequency below this fraction are condensed.
+        Takes precedence over max_categories.
+    """
+
+    def __init__(
+        self,
+        max_categories: int = 10,
+        other_label: str = "Other",
+        min_frequency: float | None = None,
+    ):
+        self.max_categories = max_categories
+        self.other_label = other_label
+        self.min_frequency = min_frequency
+
+    def fit(self, X: pd.DataFrame, y=None):
+        self.keep_categories_ = {}
+        for col in X.columns:
+            if X[col].dtype == "object" or isinstance(X[col].dtype, pd.CategoricalDtype):
+                counts = X[col].value_counts(normalize=True)
+                if self.min_frequency is not None:
+                    keep = counts[counts >= self.min_frequency].index.tolist()
+                else:
+                    keep = counts.head(self.max_categories).index.tolist()
+                self.keep_categories_[col] = set(keep)
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        Xt = X.copy()
+        for col, keep in self.keep_categories_.items():
+            if col in Xt.columns:
+                Xt[col] = Xt[col].where(Xt[col].isin(keep), self.other_label)
+        return Xt

xplainable_preprocessing/transformers/datetime_extract.py

@@ -0,0 +1,72 @@
+"""Datetime feature extraction transformer."""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+
+class DateTimeExtractTransformer(BaseEstimator, TransformerMixin):
+    """Extract datetime components as new columns.
+
+    Parameters
+    ----------
+    components : list[str]
+        Components to extract. Options:
+        "year", "month", "day", "hour", "minute", "second",
+        "dayofweek", "dayofyear", "weekofyear", "quarter",
+        "is_weekend", "is_month_start", "is_month_end"
+    drop_original : bool
+        Whether to drop the original datetime columns. Default True.
+    """
+
+    COMPONENT_MAP = {
+        "year": lambda s: s.dt.year,
+        "month": lambda s: s.dt.month,
+        "day": lambda s: s.dt.day,
+        "hour": lambda s: s.dt.hour,
+        "minute": lambda s: s.dt.minute,
+        "second": lambda s: s.dt.second,
+        "dayofweek": lambda s: s.dt.dayofweek,
+        "dayofyear": lambda s: s.dt.dayofyear,
+        "weekofyear": lambda s: s.dt.isocalendar().week.astype(int),
+        "quarter": lambda s: s.dt.quarter,
+        "is_weekend": lambda s: s.dt.dayofweek.isin([5, 6]).astype(int),
+        "is_month_start": lambda s: s.dt.is_month_start.astype(int),
+        "is_month_end": lambda s: s.dt.is_month_end.astype(int),
+    }
+
+    def __init__(
+        self,
+        components: list[str] | None = None,
+        drop_original: bool = True,
+    ):
+        self.components = components or ["year", "month", "day"]
+        self.drop_original = drop_original
+
+    def fit(self, X: pd.DataFrame, y=None):
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        Xt = X.copy()
+        original_cols = list(Xt.columns)
+
+        for col in original_cols:
+            if not pd.api.types.is_datetime64_any_dtype(Xt[col]):
+                try:
+                    Xt[col] = pd.to_datetime(Xt[col], errors="coerce")
+                except Exception:
+                    continue
+
+            if not pd.api.types.is_datetime64_any_dtype(Xt[col]):
+                continue
+
+            for component in self.components:
+                extractor = self.COMPONENT_MAP.get(component)
+                if extractor:
+                    Xt[f"{col}_{component}"] = extractor(Xt[col])
+
+            if self.drop_original:
+                Xt = Xt.drop(columns=[col])
+
+        return Xt

xplainable_preprocessing/transformers/drop_columns.py

@@ -0,0 +1,25 @@
+"""Drop columns transformer."""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+
+class DropColumnsTransformer(BaseEstimator, TransformerMixin):
+    """Drop specified columns from a DataFrame.
+
+    Parameters
+    ----------
+    columns : list[str]
+        Columns to drop.
+    """
+
+    def __init__(self, columns: list[str] | None = None):
+        self.columns = columns or []
+
+    def fit(self, X: pd.DataFrame, y=None):
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        return X.drop(columns=self.columns, errors="ignore")

xplainable_preprocessing/transformers/expression.py

@@ -0,0 +1,35 @@
+"""Expression-based transformer using pandas.eval()."""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+
+class ExpressionTransformer(BaseEstimator, TransformerMixin):
+    """Create new columns using pandas.eval() expressions.
+
+    Uses pandas.eval() for safe expression evaluation without exec/eval.
+
+    Parameters
+    ----------
+    expression : str
+        A pandas-compatible expression (e.g. "age * salary / 1000").
+    output_column : str
+        Name of the new column to create.
+    """
+
+    def __init__(self, expression: str = "", output_column: str = ""):
+        self.expression = expression
+        self.output_column = output_column
+
+    def fit(self, X: pd.DataFrame, y=None):
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        Xt = X.copy()
+        # Support both bare column names ("age * salary") and df reference ("df['age'] * df['salary']")
+        local_dict = {col: Xt[col] for col in Xt.columns}
+        local_dict["df"] = Xt
+        Xt[self.output_column] = pd.eval(self.expression, local_dict=local_dict, engine="python")
+        return Xt

xplainable_preprocessing/transformers/fill_missing.py

@@ -0,0 +1,71 @@
+"""DataFrame-aware missing value fill transformer."""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+
+class FillMissingTransformer(BaseEstimator, TransformerMixin):
+    """Fill missing values with per-column strategies.
+
+    Parameters
+    ----------
+    strategies : dict[str, str | int | float]
+        Column name -> fill strategy. Strategy can be:
+        - "mean": fill with column mean (numeric only)
+        - "median": fill with column median (numeric only)
+        - "mode": fill with most frequent value
+        - "ffill": forward fill
+        - "bfill": backward fill
+        - Any scalar value: fill with that constant
+    default : str | int | float | None
+        Default strategy for columns not in strategies dict.
+    """
+
+    def __init__(
+        self,
+        strategies: dict | None = None,
+        default: str | int | float | None = None,
+    ):
+        self.strategies = strategies or {}
+        self.default = default
+
+    def fit(self, X: pd.DataFrame, y=None):
+        self.fill_values_ = {}
+        for col, strategy in self.strategies.items():
+            if col not in X.columns:
+                continue
+            if strategy == "mean":
+                self.fill_values_[col] = X[col].mean()
+            elif strategy == "median":
+                self.fill_values_[col] = X[col].median()
+            elif strategy == "mode":
+                mode = X[col].mode()
+                self.fill_values_[col] = mode.iloc[0] if len(mode) > 0 else None
+            # ffill/bfill and constants don't need fitting
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        Xt = X.copy()
+        for col, strategy in self.strategies.items():
+            if col not in Xt.columns:
+                continue
+            if strategy in ("mean", "median", "mode"):
+                Xt[col] = Xt[col].fillna(self.fill_values_.get(col))
+            elif strategy == "ffill":
+                Xt[col] = Xt[col].ffill()
+            elif strategy == "bfill":
+                Xt[col] = Xt[col].bfill()
+            else:
+                # Treat as constant value
+                Xt[col] = Xt[col].fillna(strategy)
+
+        if self.default is not None:
+            for col in Xt.columns:
+                if col not in self.strategies:
+                    if self.default in ("ffill", "bfill"):
+                        Xt[col] = getattr(Xt[col], self.default)()
+                    else:
+                        Xt[col] = Xt[col].fillna(self.default)
+        return Xt

xplainable_preprocessing/transformers/grouped_lag.py

@@ -0,0 +1,55 @@
+"""Grouped lag feature transformer."""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+
+class GroupedLagTransformer(BaseEstimator, TransformerMixin):
+    """Create lag features within groups.
+
+    Parameters
+    ----------
+    columns : list[str]
+        Columns to create lag features for.
+    group_by : list[str]
+        Columns to group by.
+    order_by : str
+        Column to sort by within groups before computing lags.
+    periods : list[int]
+        Lag periods to create. E.g. [1, 7, 30].
+    """
+
+    def __init__(
+        self,
+        columns: list[str] | None = None,
+        group_by: list[str] | None = None,
+        order_by: str = "",
+        periods: list[int] | None = None,
+    ):
+        self.columns = columns or []
+        self.group_by = group_by or []
+        self.order_by = order_by
+        self.periods = periods or [1]
+
+    def fit(self, X: pd.DataFrame, y=None):
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        Xt = X.copy()
+
+        if self.order_by and self.order_by in Xt.columns:
+            Xt = Xt.sort_values(self.group_by + [self.order_by])
+
+        for col in self.columns:
+            if col not in Xt.columns:
+                continue
+            for period in self.periods:
+                new_col = f"{col}_lag_{period}"
+                if self.group_by:
+                    Xt[new_col] = Xt.groupby(self.group_by)[col].shift(period)
+                else:
+                    Xt[new_col] = Xt[col].shift(period)
+
+        return Xt

xplainable_preprocessing/transformers/rename_columns.py

@@ -0,0 +1,25 @@
+"""Rename columns transformer."""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+
+class RenameColumnsTransformer(BaseEstimator, TransformerMixin):
+    """Rename columns in a DataFrame.
+
+    Parameters
+    ----------
+    mapping : dict[str, str]
+        Old name -> new name mapping.
+    """
+
+    def __init__(self, mapping: dict[str, str] | None = None):
+        self.mapping = mapping or {}
+
+    def fit(self, X: pd.DataFrame, y=None):
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        return X.rename(columns=self.mapping)

xplainable_preprocessing/transformers/rolling_agg.py

@@ -0,0 +1,79 @@
+"""Rolling window aggregation transformer."""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+
+class RollingAggTransformer(BaseEstimator, TransformerMixin):
+    """Create rolling window aggregate features.
+
+    Parameters
+    ----------
+    columns : list[str]
+        Columns to compute rolling aggregates for.
+    group_by : list[str] | None
+        Columns to group by. If None, computes globally.
+    window : int
+        Rolling window size.
+    operation : str
+        Aggregation operation: "mean", "sum", "min", "max", "std", "count".
+    min_periods : int
+        Minimum number of observations required. Default 1.
+    order_by : str
+        Column to sort by before computing rolling stats.
+    """
+
+    OPERATIONS = {"mean", "sum", "min", "max", "std", "count", "median"}
+
+    def __init__(
+        self,
+        columns: list[str] | None = None,
+        group_by: list[str] | None = None,
+        window: int = 7,
+        operation: str = "mean",
+        min_periods: int = 1,
+        order_by: str = "",
+    ):
+        self.columns = columns or []
+        self.group_by = group_by
+        self.window = window
+        self.operation = operation
+        self.min_periods = min_periods
+        self.order_by = order_by
+
+    def fit(self, X: pd.DataFrame, y=None):
+        if self.operation not in self.OPERATIONS:
+            raise ValueError(
+                f"Unknown operation '{self.operation}'. "
+                f"Available: {sorted(self.OPERATIONS)}"
+            )
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        Xt = X.copy()
+
+        if self.order_by and self.order_by in Xt.columns:
+            sort_cols = (self.group_by or []) + [self.order_by]
+            Xt = Xt.sort_values(sort_cols)
+
+        for col in self.columns:
+            if col not in Xt.columns:
+                continue
+            new_col = f"{col}_rolling_{self.operation}_{self.window}"
+            if self.group_by:
+                rolling = (
+                    Xt.groupby(self.group_by)[col]
+                    .rolling(window=self.window, min_periods=self.min_periods)
+                )
+                Xt[new_col] = getattr(rolling, self.operation)().reset_index(
+                    level=list(range(len(self.group_by))), drop=True
+                )
+            else:
+                rolling = Xt[col].rolling(
+                    window=self.window, min_periods=self.min_periods
+                )
+                Xt[new_col] = getattr(rolling, self.operation)()
+
+        return Xt

xplainable_preprocessing/transformers/text_clean.py

@@ -0,0 +1,70 @@
+"""Text cleaning transformer."""
+
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+
+class TextCleanTransformer(BaseEstimator, TransformerMixin):
+    """Clean text columns with configurable operations.
+
+    Parameters
+    ----------
+    operations : list[str]
+        Ordered list of operations to apply. Options:
+        - "lowercase": convert to lowercase
+        - "uppercase": convert to uppercase
+        - "strip": strip whitespace
+        - "remove_digits": remove all digits
+        - "remove_punctuation": remove punctuation
+        - "remove_extra_whitespace": collapse multiple spaces
+        - "remove_html": strip HTML tags
+    regex_pattern : str | None
+        Optional regex pattern to remove.
+    regex_replacement : str
+        Replacement for regex matches. Default "".
+    """
+
+    def __init__(
+        self,
+        operations: list[str] | None = None,
+        regex_pattern: str | None = None,
+        regex_replacement: str = "",
+    ):
+        self.operations = operations or []
+        self.regex_pattern = regex_pattern
+        self.regex_replacement = regex_replacement
+
+    def fit(self, X: pd.DataFrame, y=None):
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        Xt = X.copy()
+        for col in Xt.columns:
+            if Xt[col].dtype != "object":
+                continue
+            series = Xt[col]
+            for op in self.operations:
+                if op == "lowercase":
+                    series = series.str.lower()
+                elif op == "uppercase":
+                    series = series.str.upper()
+                elif op == "strip":
+                    series = series.str.strip()
+                elif op == "remove_digits":
+                    series = series.str.replace(r"\d+", "", regex=True)
+                elif op == "remove_punctuation":
+                    series = series.str.replace(r"[^\w\s]", "", regex=True)
+                elif op == "remove_extra_whitespace":
+                    series = series.str.replace(r"\s+", " ", regex=True).str.strip()
+                elif op == "remove_html":
+                    series = series.str.replace(r"<[^>]+>", "", regex=True)
+            if self.regex_pattern:
+                series = series.str.replace(
+                    self.regex_pattern, self.regex_replacement, regex=True
+                )
+            Xt[col] = series
+        return Xt

xplainable_preprocessing/transformers/type_cast.py

@@ -0,0 +1,42 @@
+"""Type casting transformer."""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+
+class TypeCastTransformer(BaseEstimator, TransformerMixin):
+    """Cast column dtypes in a DataFrame.
+
+    Parameters
+    ----------
+    dtypes : dict[str, str]
+        Column name -> target dtype mapping.
+        E.g. {"age": "float64", "name": "string"}
+    errors : str
+        How to handle casting errors. Default "coerce" (invalid -> NaN).
+    """
+
+    def __init__(self, dtypes: dict[str, str] | None = None, errors: str = "coerce"):
+        self.dtypes = dtypes or {}
+        self.errors = errors
+
+    def fit(self, X: pd.DataFrame, y=None):
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        Xt = X.copy()
+        for col, dtype in self.dtypes.items():
+            if col in Xt.columns:
+                if dtype in ("int", "int64", "int32", "float", "float64", "float32"):
+                    Xt[col] = pd.to_numeric(Xt[col], errors=self.errors)
+                    if dtype.startswith("int") and self.errors == "coerce":
+                        Xt[col] = Xt[col].astype("Int64")
+                    else:
+                        Xt[col] = Xt[col].astype(dtype)
+                elif dtype in ("datetime", "datetime64[ns]"):
+                    Xt[col] = pd.to_datetime(Xt[col], errors=self.errors)
+                else:
+                    Xt[col] = Xt[col].astype(dtype)
+        return Xt

xplainable_preprocessing-0.1.0.dist-info/METADATA

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: xplainable-preprocessing
+Version: 0.1.0
+Summary: Shared preprocessing pipeline package for xplainable
+Requires-Python: >=3.9
+Requires-Dist: cloudpickle>=3.0
+Requires-Dist: numpy>=1.24
+Requires-Dist: pandas>=2.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: scikit-learn>=1.3
+Provides-Extra: dev
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'

xplainable_preprocessing-0.1.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+xplainable_preprocessing/__init__.py,sha256=2y36kqaTg6eZNDqX6inCTOJIg8eh2uB3CIvBM3WBMM4,688
+xplainable_preprocessing/compiler.py,sha256=AlyPGGdrcJ-4zHGkrNnaYRC_NimgrXSL5L82YXrlmBQ,2172
+xplainable_preprocessing/pipeline.py,sha256=Anve7vAeLRGqr1HR1OfHrupNexHedz8GvBlj_3jK028,3161
+xplainable_preprocessing/preview.py,sha256=kOzGPPvNTmH65Je9urLuTKVgEy9lbLWJsYHgqxZs8u8,3662
+xplainable_preprocessing/registry.py,sha256=MYvj7jiBpk3WL6BoWzuJdOTOOy_8CldZo0oJyRaiJOI,3325
+xplainable_preprocessing/sandbox.py,sha256=e0SkK6atbioDw5mcIedije0CC00YdnjlrTgg8Bd-GB4,4308
+xplainable_preprocessing/schema.py,sha256=EhqBJCIKAK7j93tJ3YgjK92qV8budXHzBfH6H04VTAs,2220
+xplainable_preprocessing/serialization.py,sha256=tyom9Wlbpre8ryAjjGUNuwyvh36qeyrv-JJHmlGror8,551
+xplainable_preprocessing/transformers/__init__.py,sha256=eTiTgQu_OeQ3KjmjhH7g4g3T07u67tF2B6CFT1U_ahU,1183
+xplainable_preprocessing/transformers/category_condense.py,sha256=adaXy6_zb4Iq87zar7yHZ8Pb2lR1r3xUo-_h7TvMsDA,1777
+xplainable_preprocessing/transformers/datetime_extract.py,sha256=q2Yp6V8jX3yC1N9mcJRaNuUFOGqF5QEsOxpw8mg36CY,2395
+xplainable_preprocessing/transformers/drop_columns.py,sha256=K8A2b2wWAGtizFs3KPsrP-4TctS_oJ5viRMXSWZMDwY,628
+xplainable_preprocessing/transformers/expression.py,sha256=xjXL-pN8ypqX3bWbTMS-nTKIcjPqYaCj7cr6imutrPM,1168
+xplainable_preprocessing/transformers/fill_missing.py,sha256=3vP-uK5MIIWwRbx4eDmfjocxmfJfDIjliH2eaJOCO5U,2569
+xplainable_preprocessing/transformers/grouped_lag.py,sha256=Q0KCSdLERJZoUPI26oNkwSovEsz5ua8fONYNulmiFow,1581
+xplainable_preprocessing/transformers/rename_columns.py,sha256=HYEurKTxa2n4IZ1Ta9DGwCeV6hsr9NgqJYKAjORoM-s,630
+xplainable_preprocessing/transformers/rolling_agg.py,sha256=y_lz2Qkj2JVTcap4mRkTB_bnfrsing5fCGhxXZ_nnP4,2588
+xplainable_preprocessing/transformers/text_clean.py,sha256=At1olQy0mBSQt65QPJnGlPAUQDEIsWbrBe0rYqloJmg,2461
+xplainable_preprocessing/transformers/type_cast.py,sha256=l7DIrAAI0Ds2Eyng4gIctwy3mn1K3AqZAesa_lW_FMk,1483
+xplainable_preprocessing-0.1.0.dist-info/METADATA,sha256=cNlOGuS5DJQp_geFD3hYnIrKKmFR-EBMa-9df4LrUW4,406
+xplainable_preprocessing-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+xplainable_preprocessing-0.1.0.dist-info/RECORD,,

xplainable_preprocessing-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any