xplainable-preprocessing 0.1.0__tar.gz → 0.2.1__tar.gz

xplainable_preprocessing-0.2.1/.github/workflows/publish-pypi.yml

@@ -0,0 +1,42 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+
+    - name: Install build tools
+      run: pip install build
+
+    - name: Build package
+      run: python -m build
+
+    - name: Publish to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.PYPI_API_TOKEN }}
+
+    - name: Create GitHub Release
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        VERSION=${GITHUB_REF_NAME}
+        gh release create "$VERSION" dist/* \
+          --title "$VERSION" \
+          --generate-notes \
+          --latest

xplainable_preprocessing-0.2.1/.gitignore

@@ -0,0 +1 @@
+.env

{xplainable_preprocessing-0.1.0 → xplainable_preprocessing-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xplainable-preprocessing
-Version: 0.1.0
+Version: 0.2.1
 Summary: Shared preprocessing pipeline package for xplainable
 Requires-Python: >=3.9
 Requires-Dist: cloudpickle>=3.0

xplainable_preprocessing-0.2.1/README.md

@@ -0,0 +1,164 @@
+# xplainable-preprocessing
+
+Shared preprocessing pipeline library for [xplainable](https://xplainable.io). Define, compile, fit, serialise, and apply data transformation pipelines using a declarative JSON spec.
+
+## Install
+
+```bash
+pip install xplainable-preprocessing
+```
+
+## Quick Start
+
+### Define a pipeline as JSON
+
+```python
+from xplainable_preprocessing.schema import PipelineSpec
+from xplainable_preprocessing.compiler import compile_spec
+
+spec = PipelineSpec(
+    version="2.0",
+    steps=[
+        {"id": "drop_ids", "type": "DropColumnsTransformer", "params": {"columns": ["customer_id", "row_id"]}},
+        {"id": "fill_numeric", "type": "FillMissingTransformer", "columns": ["tenure", "charges"], "params": {"strategies": {"tenure": "median", "charges": "median"}}},
+        {"id": "clip_outliers", "type": "ClipTransformer", "columns": ["charges"], "params": {"min_val": 0, "max_val": 500}},
+        {"id": "missing_flags", "type": "MissingFlagTransformer", "columns": ["tenure", "charges"]},
+        {"id": "condense_plan", "type": "CategoryCondenseTransformer", "columns": ["plan_type"], "params": {"max_categories": 10}},
+        {"id": "extract_dates", "type": "DateTimeExtractTransformer", "columns": ["signup_date"], "params": {"components": ["month", "dayofweek"], "drop_original": true}},
+    ]
+)
+
+pipeline = compile_spec(spec)
+```
+
+### Fit and transform
+
+```python
+import pandas as pd
+
+df = pd.read_csv("data.csv")
+pipeline.fit(df)
+transformed = pipeline.transform(df)
+```
+
+### Serialise and load
+
+```python
+from xplainable_preprocessing.serialization import save_pipeline, load_pipeline
+
+# Save
+binary = save_pipeline(pipeline)
+
+# Load
+loaded = load_pipeline(binary)
+result = loaded.transform(new_data)
+```
+
+## Available Transformers
+
+### Data Cleaning
+| Transformer | Description |
+|-------------|-------------|
+| `DropColumnsTransformer` | Drop specified columns |
+| `RenameColumnsTransformer` | Rename columns |
+| `TypeCastTransformer` | Cast column dtypes (e.g. str to float64, with error coercion) |
+| `ClipTransformer` | Clip numeric values outside (min, max) to NaN |
+
+### Missing Values
+| Transformer | Description |
+|-------------|-------------|
+| `FillMissingTransformer` | Fill nulls with median, mode, constant, or other strategies |
+| `MissingFlagTransformer` | Create binary 0/1 indicator columns for nulls |
+
+### Feature Engineering
+| Transformer | Description |
+|-------------|-------------|
+| `ExpressionTransformer` | Create derived columns via pandas expressions |
+| `DateTimeExtractTransformer` | Extract year, month, dayofweek, etc. from datetime columns |
+| `GroupByAggTransformer` | Aggregate features by group (count, sum, mean) |
+| `GroupedLagTransformer` | Create lagged features within groups |
+| `RollingAggTransformer` | Rolling window aggregations |
+
+### Categorical
+| Transformer | Description |
+|-------------|-------------|
+| `CategoryCondenseTransformer` | Condense high-cardinality categoricals to top N + "Other" |
+| `TextCleanTransformer` | Clean text: lowercase, strip, remove HTML/extra whitespace |
+
+### Scaling (use with caution -- breaks xplainable model explainability)
+| Transformer | Description |
+|-------------|-------------|
+| `StandardScaler` | Standardise to zero mean, unit variance |
+| `MinMaxScaler` | Scale to [0, 1] range |
+| `RobustScaler` | Scale using median and IQR |
+| `PowerTransformer` | Apply power transform for normality |
+| `QuantileTransformer` | Transform to uniform or normal distribution |
+
+> **Note:** Scaling transformers are available but should NOT be used with xplainable models. xplainable models are inherently explainable and require raw feature values. Scaling destroys interpretability.
+
+### Other sklearn
+| Transformer | Description |
+|-------------|-------------|
+| `SimpleImputer` | sklearn's imputer |
+| `OneHotEncoder` | One-hot encode categoricals |
+| `OrdinalEncoder` | Ordinal encode categoricals |
+| `KBinsDiscretizer` | Discretise continuous features into bins |
+| `Binarizer` | Threshold features to binary |
+
+## Pipeline Spec Format
+
+Pipelines are defined as JSON for portability across the xplainable platform, MCP tools, and API:
+
+```json
+{
+    "version": "2.0",
+    "steps": [
+        {
+            "id": "step_name",
+            "type": "TransformerName",
+            "columns": ["col1", "col2"],
+            "params": {"param1": "value1"},
+            "description": "Optional description"
+        }
+    ]
+}
+```
+
+- `id`: unique step identifier
+- `type`: transformer class name (from the registry)
+- `columns`: optional list of columns to apply to (wraps in `DataFrameColumnTransformer`)
+- `params`: constructor arguments for the transformer
+- `description`: optional human-readable description
+
+## Architecture
+
+```
+PipelineSpec (JSON)
+    ↓ compile_spec()
+DataFramePipeline (sklearn Pipeline of transformers)
+    ↓ fit() / transform()
+Transformed DataFrame
+    ↓ save_pipeline()
+Binary (cloudpickle)
+    ↓ load_pipeline()
+DataFramePipeline (restored)
+```
+
+Key modules:
+- `schema.py` -- Pydantic models for `PipelineSpec` and `StepSpec`
+- `compiler.py` -- compiles JSON spec into sklearn pipeline
+- `pipeline.py` -- `DataFramePipeline` wrapper around sklearn `Pipeline`
+- `registry.py` -- maps transformer names to classes, generates LLM-ready catalog
+- `serialization.py` -- cloudpickle save/load
+- `preview.py` -- before/after preview with schema deltas
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+MIT

{xplainable_preprocessing-0.1.0 → xplainable_preprocessing-0.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "xplainable-preprocessing"
-version = "0.1.0"
+version = "0.2.1"
 description = "Shared preprocessing pipeline package for xplainable"
 requires-python = ">=3.9"
 dependencies = [

{xplainable_preprocessing-0.1.0 → xplainable_preprocessing-0.2.1}/src/xplainable_preprocessing/registry.py

@@ -20,11 +20,14 @@ from sklearn.preprocessing import (
 
 from xplainable_preprocessing.transformers import (
     CategoryCondenseTransformer,
+    ClipTransformer,
     DateTimeExtractTransformer,
     DropColumnsTransformer,
     ExpressionTransformer,
     FillMissingTransformer,
+    GroupByAggTransformer,
     GroupedLagTransformer,
+    MissingFlagTransformer,
     RenameColumnsTransformer,
     RollingAggTransformer,
     TextCleanTransformer,
@@ -55,6 +58,9 @@ REGISTRY: dict[str, type] = {
     "DateTimeExtractTransformer": DateTimeExtractTransformer,
     "GroupedLagTransformer": GroupedLagTransformer,
     "RollingAggTransformer": RollingAggTransformer,
+    "GroupByAggTransformer": GroupByAggTransformer,
+    "ClipTransformer": ClipTransformer,
+    "MissingFlagTransformer": MissingFlagTransformer,
 }
 
 

xplainable_preprocessing-0.2.1/src/xplainable_preprocessing/schema.py

@@ -0,0 +1,372 @@
+"""Pydantic models for pipeline specification with mutation support."""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, field_validator
+
+
+class StepSpec(BaseModel):
+    """A single preprocessing step in the pipeline.
+
+    The ``outputs``, ``modifies``, and ``drops`` fields form the step's
+    **column contract**.  They are populated after the first fit via
+    ``PipelineSpec.enrich_from_deltas()`` and enable cascade analysis
+    for step mutations (remove, reorder) without recompiling the pipeline.
+    """
+
+    id: str
+    type: str
+    columns: Optional[List[str]] = None
+    params: Dict = {}
+    description: Optional[str] = None
+
+    # Column contract — populated after first fit via enrich_from_deltas()
+    outputs: Optional[List[str]] = None   # columns this step CREATES
+    modifies: Optional[List[str]] = None  # columns this step CHANGES in place
+    drops: Optional[List[str]] = None     # columns this step REMOVES
+
+    @field_validator("id")
+    @classmethod
+    def id_must_be_non_empty(cls, v: str) -> str:
+        if not v.strip():
+            raise ValueError("Step id must be non-empty")
+        return v
+
+    @field_validator("type")
+    @classmethod
+    def type_must_be_non_empty(cls, v: str) -> str:
+        if not v.strip():
+            raise ValueError("Step type must be non-empty")
+        return v
+
+
+class RemovalAnalysis(BaseModel):
+    """Result of analyzing what happens when a step is removed."""
+
+    step_id: str
+    cascade_remove: List[str] = []       # step IDs that must also be removed
+    columns_lost: List[str] = []         # columns that would disappear
+    columns_restored: List[str] = []     # columns that were dropped and would reappear
+    safe: bool = True                    # True if no downstream steps are affected
+
+
+class PipelineSpec(BaseModel):
+    """Full pipeline specification containing ordered steps.
+
+    Supports mutation methods (``remove_step``, ``insert_step``,
+    ``reorder_step``) that return new ``PipelineSpec`` instances.
+    Cascade analysis via ``analyze_removal`` works from the column
+    contract without needing to recompile the pipeline.
+    """
+
+    version: str = "2.0"
+    steps: List[StepSpec] = []
+
+    @field_validator("steps")
+    @classmethod
+    def step_ids_must_be_unique(cls, v: List[StepSpec]) -> List[StepSpec]:
+        ids = [step.id for step in v]
+        if len(ids) != len(set(ids)):
+            duplicates = [id_ for id_ in ids if ids.count(id_) > 1]
+            raise ValueError(f"Duplicate step ids: {set(duplicates)}")
+        return v
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    def get_step(self, step_id: str) -> Optional[StepSpec]:
+        """Return a step by ID, or None."""
+        for step in self.steps:
+            if step.id == step_id:
+                return step
+        return None
+
+    def step_index(self, step_id: str) -> int:
+        """Return the index of a step by ID. Raises ValueError if not found."""
+        for i, step in enumerate(self.steps):
+            if step.id == step_id:
+                return i
+        raise ValueError(f"Step '{step_id}' not found in spec")
+
+    def steps_after(self, step_id: str) -> List[StepSpec]:
+        """Return all steps that come after the given step."""
+        idx = self.step_index(step_id)
+        return self.steps[idx + 1:]
+
+    @property
+    def is_enriched(self) -> bool:
+        """True if column contracts have been populated on all steps."""
+        return all(
+            step.outputs is not None
+            for step in self.steps
+        )
+
+    # ------------------------------------------------------------------
+    # Enrichment — write column contracts from deltas
+    # ------------------------------------------------------------------
+
+    def enrich_from_deltas(self, step_deltas: list[dict]) -> "PipelineSpec":
+        """Return a new spec with column contracts populated from step deltas.
+
+        Parameters
+        ----------
+        step_deltas : list[dict]
+            Output of ``preview.compute_step_deltas()``.  Each entry has
+            ``step_id`` and ``delta: {added, updated, dropped}``.
+
+        Returns
+        -------
+        PipelineSpec
+            A new spec instance with ``outputs``/``modifies``/``drops``
+            populated on each step.
+        """
+        delta_map = {d["step_id"]: d for d in step_deltas}
+        enriched_steps = []
+        for step in self.steps:
+            entry = delta_map.get(step.id, {})
+            delta = entry.get("delta", {})
+            updates = {
+                "outputs": delta.get("added", []),
+                "modifies": delta.get("updated", []),
+                "drops": delta.get("dropped", []),
+            }
+            # columns (inputs) is left as-is — only the LLM-declared value
+            # is used for dependency resolution. Outputs/modifies/drops are
+            # sufficient for cascade analysis.
+            enriched_steps.append(step.model_copy(update=updates))
+        return PipelineSpec(version=self.version, steps=enriched_steps)
+
+    # ------------------------------------------------------------------
+    # Cascade analysis
+    # ------------------------------------------------------------------
+
+    def analyze_removal(self, step_id: str) -> RemovalAnalysis:
+        """Analyze what happens if a step is removed.
+
+        Uses the column contract (``outputs``/``drops``) to determine
+        which downstream steps would lose their input columns and must
+        also be removed (cascade).
+
+        Works without recompiling the pipeline — purely from the spec
+        metadata.  Requires the spec to be enriched first.
+
+        Parameters
+        ----------
+        step_id : str
+            The ID of the step to analyze removing.
+
+        Returns
+        -------
+        RemovalAnalysis
+        """
+        step = self.get_step(step_id)
+        if step is None:
+            raise ValueError(f"Step '{step_id}' not found")
+
+        # Columns that would disappear if this step is removed
+        lost_columns = set(step.outputs or [])
+
+        # Walk downstream and find cascade
+        cascade = []
+        for other in self.steps_after(step_id):
+            inputs = set(other.columns or [])
+            if inputs & lost_columns:
+                cascade.append(other.id)
+                # This step's outputs are also lost in the cascade
+                lost_columns.update(other.outputs or [])
+
+        # Columns that this step drops would be restored
+        restored = list(step.drops or [])
+
+        return RemovalAnalysis(
+            step_id=step_id,
+            cascade_remove=cascade,
+            columns_lost=sorted(lost_columns),
+            columns_restored=sorted(restored),
+            safe=len(cascade) == 0,
+        )
+
+    # ------------------------------------------------------------------
+    # Mutations — all return NEW PipelineSpec instances (immutable pattern)
+    # ------------------------------------------------------------------
+
+    def remove_step(self, step_id: str, cascade: bool = True) -> "PipelineSpec":
+        """Return a new spec with the step (and optionally its cascade) removed.
+
+        Parameters
+        ----------
+        step_id : str
+            The step to remove.
+        cascade : bool
+            If True, also remove downstream steps that depend on columns
+            created by this step.  If False, only remove this single step
+            (caller is responsible for ensuring validity).
+
+        Returns
+        -------
+        PipelineSpec
+        """
+        analysis = self.analyze_removal(step_id)
+        remove_ids = {step_id}
+        if cascade:
+            remove_ids.update(analysis.cascade_remove)
+        return PipelineSpec(
+            version=self.version,
+            steps=[s for s in self.steps if s.id not in remove_ids],
+        )
+
+    def insert_step(
+        self,
+        step: StepSpec,
+        after: Optional[str] = None,
+    ) -> "PipelineSpec":
+        """Return a new spec with a step inserted.
+
+        Parameters
+        ----------
+        step : StepSpec
+            The step to insert.
+        after : str or None
+            Insert after this step ID.  If None, append to the end.
+
+        Returns
+        -------
+        PipelineSpec
+        """
+        new_steps = list(self.steps)
+        if after is None:
+            new_steps.append(step)
+        else:
+            idx = self.step_index(after)
+            new_steps.insert(idx + 1, step)
+        return PipelineSpec(version=self.version, steps=new_steps)
+
+    def reorder_step(self, step_id: str, new_index: int) -> "PipelineSpec":
+        """Return a new spec with a step moved to a new position.
+
+        Parameters
+        ----------
+        step_id : str
+            The step to move.
+        new_index : int
+            The target index (0-based).
+
+        Returns
+        -------
+        PipelineSpec
+        """
+        idx = self.step_index(step_id)
+        step = self.steps[idx]
+        remaining = [s for s in self.steps if s.id != step_id]
+        clamped = max(0, min(new_index, len(remaining)))
+        remaining.insert(clamped, step)
+        return PipelineSpec(version=self.version, steps=remaining)
+
+    def update_step_params(self, step_id: str, params: Dict) -> "PipelineSpec":
+        """Return a new spec with a step's params updated.
+
+        Parameters
+        ----------
+        step_id : str
+            The step to update.
+        params : dict
+            New params to merge into the step's existing params.
+
+        Returns
+        -------
+        PipelineSpec
+        """
+        new_steps = []
+        for step in self.steps:
+            if step.id == step_id:
+                merged = {**step.params, **params}
+                new_steps.append(step.model_copy(update={"params": merged}))
+            else:
+                new_steps.append(step)
+        return PipelineSpec(version=self.version, steps=new_steps)
+
+
+    def optimize(self) -> "PipelineSpec":
+        """Return a new spec with steps in optimal execution order.
+
+        Topologically sorts steps so that any step producing a column
+        comes before steps that consume it.  Independent steps retain
+        their relative order.
+
+        Requires column contracts (``outputs``) and declared ``columns``
+        (inputs) on each step.  If the spec is not enriched, returns
+        a copy with the original order.
+        """
+        if not self.is_enriched:
+            return self.model_copy()
+
+        from collections import defaultdict
+
+        step_map = {s.id: s for s in self.steps}
+        ids = [s.id for s in self.steps]
+
+        # Build producer map: column → step that creates it
+        producers: dict[str, str] = {}
+        for s in self.steps:
+            for col in s.outputs or []:
+                if col not in producers:
+                    producers[col] = s.id
+
+        # Build adjacency from declared inputs
+        adj: dict[str, set[str]] = defaultdict(set)
+        in_deg: dict[str, int] = {sid: 0 for sid in ids}
+        for s in self.steps:
+            for col in s.columns or []:
+                prod = producers.get(col)
+                if prod and prod != s.id and prod in step_map:
+                    if prod not in adj or s.id not in adj[prod]:
+                        adj[prod].add(s.id)
+                        in_deg[s.id] += 1
+
+        # Kahn's algorithm — stable (preserves original order within tiers)
+        ordered: list[str] = []
+        remaining = set(ids)
+        while remaining:
+            ready = [sid for sid in ids if sid in remaining and in_deg.get(sid, 0) == 0]
+            if not ready:
+                ordered.extend(sorted(remaining))
+                break
+            ordered.extend(ready)
+            for sid in ready:
+                remaining.discard(sid)
+                for nb in adj.get(sid, set()):
+                    in_deg[nb] = max(in_deg.get(nb, 1) - 1, 0)
+
+        return PipelineSpec(
+            version=self.version,
+            steps=[step_map[sid] for sid in ordered if sid in step_map],
+        )
+
+
+def validate_spec(spec: PipelineSpec) -> None:
+    """Validate a PipelineSpec. Raises ValueError if invalid.
+
+    Checks beyond Pydantic validation:
+    - All non-custom types must exist in the registry
+    - Custom steps must have 'code' and 'class_name' in params
+    """
+    from xplainable_preprocessing.registry import REGISTRY
+
+    for step in spec.steps:
+        if step.type == "custom":
+            if "code" not in step.params:
+                raise ValueError(
+                    f"Step '{step.id}': custom type requires 'code' in params"
+                )
+            if "class_name" not in step.params:
+                raise ValueError(
+                    f"Step '{step.id}': custom type requires 'class_name' in params"
+                )
+        elif step.type not in REGISTRY:
+            raise ValueError(
+                f"Step '{step.id}': unknown type '{step.type}'. "
+                f"Available: {sorted(REGISTRY.keys())}"
+            )

{xplainable_preprocessing-0.1.0 → xplainable_preprocessing-0.2.1}/src/xplainable_preprocessing/transformers/__init__.py

@@ -1,23 +1,29 @@
-from xplainable_preprocessing.transformers.expression import ExpressionTransformer
-from xplainable_preprocessing.transformers.drop_columns import DropColumnsTransformer
-from xplainable_preprocessing.transformers.rename_columns import RenameColumnsTransformer
-from xplainable_preprocessing.transformers.type_cast import TypeCastTransformer
-from xplainable_preprocessing.transformers.fill_missing import FillMissingTransformer
 from xplainable_preprocessing.transformers.category_condense import CategoryCondenseTransformer
-from xplainable_preprocessing.transformers.text_clean import TextCleanTransformer
+from xplainable_preprocessing.transformers.clip import ClipTransformer
 from xplainable_preprocessing.transformers.datetime_extract import DateTimeExtractTransformer
+from xplainable_preprocessing.transformers.drop_columns import DropColumnsTransformer
+from xplainable_preprocessing.transformers.expression import ExpressionTransformer
+from xplainable_preprocessing.transformers.fill_missing import FillMissingTransformer
+from xplainable_preprocessing.transformers.groupby_agg import GroupByAggTransformer
 from xplainable_preprocessing.transformers.grouped_lag import GroupedLagTransformer
+from xplainable_preprocessing.transformers.missing_flag import MissingFlagTransformer
+from xplainable_preprocessing.transformers.rename_columns import RenameColumnsTransformer
 from xplainable_preprocessing.transformers.rolling_agg import RollingAggTransformer
+from xplainable_preprocessing.transformers.text_clean import TextCleanTransformer
+from xplainable_preprocessing.transformers.type_cast import TypeCastTransformer
 
 __all__ = [
-    "ExpressionTransformer",
-    "DropColumnsTransformer",
-    "RenameColumnsTransformer",
-    "TypeCastTransformer",
-    "FillMissingTransformer",
     "CategoryCondenseTransformer",
-    "TextCleanTransformer",
+    "ClipTransformer",
     "DateTimeExtractTransformer",
+    "DropColumnsTransformer",
+    "ExpressionTransformer",
+    "FillMissingTransformer",
+    "GroupByAggTransformer",
     "GroupedLagTransformer",
+    "MissingFlagTransformer",
+    "RenameColumnsTransformer",
     "RollingAggTransformer",
+    "TextCleanTransformer",
+    "TypeCastTransformer",
 ]

xplainable_preprocessing-0.2.1/src/xplainable_preprocessing/transformers/clip.py

@@ -0,0 +1,45 @@
+"""Clip transformer -- clips numeric values outside a range to NaN."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+
+class ClipTransformer(BaseEstimator, TransformerMixin):
+    """Clip numeric values outside (min_val, max_val) to NaN.
+
+    Useful for nulling out physically impossible values
+    (e.g. BMI of 0, 2090 push-ups) so they don't create
+    junk bins in the model.
+
+    Parameters
+    ----------
+    min_val : float | None
+        Values strictly below this are set to NaN. None means no lower bound.
+    max_val : float | None
+        Values strictly above this are set to NaN. None means no upper bound.
+    """
+
+    def __init__(
+        self,
+        min_val: float | None = None,
+        max_val: float | None = None,
+    ):
+        self.min_val = min_val
+        self.max_val = max_val
+
+    def fit(self, X: pd.DataFrame, y=None):
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        X = X.copy()
+        for col in X.columns:
+            if not pd.api.types.is_numeric_dtype(X[col]):
+                continue
+            if self.min_val is not None:
+                X.loc[X[col] < self.min_val, col] = np.nan
+            if self.max_val is not None:
+                X.loc[X[col] > self.max_val, col] = np.nan
+        return X

xplainable_preprocessing-0.2.1/src/xplainable_preprocessing/transformers/missing_flag.py

@@ -0,0 +1,48 @@
+"""Missing flag transformer -- creates binary indicators for null values."""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+
+class MissingFlagTransformer(BaseEstimator, TransformerMixin):
+    """Create binary 0/1 indicator columns for null values.
+
+    For each specified column, adds a new column ``{col}_missing``
+    that is 1 where the original value is null and 0 otherwise.
+    Useful when missingness itself is informative (e.g. "physical
+    testing not completed" predicts injury risk).
+
+    Parameters
+    ----------
+    suffix : str
+        Suffix appended to column names for the flag columns.
+        Default ``"_missing"``.
+    drop_original : bool
+        If True, drop the original columns after creating flags.
+        Default False.
+    """
+
+    def __init__(
+        self,
+        suffix: str = "_missing",
+        drop_original: bool = False,
+    ):
+        self.suffix = suffix
+        self.drop_original = drop_original
+
+    def fit(self, X: pd.DataFrame, y=None):
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        X = X.copy()
+        flag_cols = {}
+        for col in X.columns:
+            flag_cols[f"{col}{self.suffix}"] = X[col].isna().astype(int)
+        flags = pd.DataFrame(flag_cols, index=X.index)
+        if self.drop_original:
+            X = pd.concat([flags], axis=1)
+        else:
+            X = pd.concat([X, flags], axis=1)
+        return X

xplainable_preprocessing-0.1.0/src/xplainable_preprocessing/schema.py

@@ -1,71 +0,0 @@
-"""Pydantic models for pipeline specification."""
-
-from typing import Dict, List, Optional
-
-from pydantic import BaseModel, field_validator
-
-
-class StepSpec(BaseModel):
-    """A single preprocessing step in the pipeline."""
-
-    id: str
-    type: str
-    columns: Optional[List[str]] = None
-    params: Dict = {}
-    description: Optional[str] = None
-
-    @field_validator("id")
-    @classmethod
-    def id_must_be_non_empty(cls, v: str) -> str:
-        if not v.strip():
-            raise ValueError("Step id must be non-empty")
-        return v
-
-    @field_validator("type")
-    @classmethod
-    def type_must_be_non_empty(cls, v: str) -> str:
-        if not v.strip():
-            raise ValueError("Step type must be non-empty")
-        return v
-
-
-class PipelineSpec(BaseModel):
-    """Full pipeline specification containing ordered steps."""
-
-    version: str = "2.0"
-    steps: List[StepSpec] = []
-
-    @field_validator("steps")
-    @classmethod
-    def step_ids_must_be_unique(cls, v: List[StepSpec]) -> List[StepSpec]:
-        ids = [step.id for step in v]
-        if len(ids) != len(set(ids)):
-            duplicates = [id_ for id_ in ids if ids.count(id_) > 1]
-            raise ValueError(f"Duplicate step ids: {set(duplicates)}")
-        return v
-
-
-def validate_spec(spec: PipelineSpec) -> None:
-    """Validate a PipelineSpec. Raises ValueError if invalid.
-
-    Checks beyond Pydantic validation:
-    - All non-custom types must exist in the registry
-    - Custom steps must have 'code' and 'class_name' in params
-    """
-    from xplainable_preprocessing.registry import REGISTRY
-
-    for step in spec.steps:
-        if step.type == "custom":
-            if "code" not in step.params:
-                raise ValueError(
-                    f"Step '{step.id}': custom type requires 'code' in params"
-                )
-            if "class_name" not in step.params:
-                raise ValueError(
-                    f"Step '{step.id}': custom type requires 'class_name' in params"
-                )
-        elif step.type not in REGISTRY:
-            raise ValueError(
-                f"Step '{step.id}': unknown type '{step.type}'. "
-                f"Available: {sorted(REGISTRY.keys())}"
-            )