xplainable-preprocessing 0.2.3__tar.gz → 0.3.0__tar.gz

xplainable_preprocessing-0.3.0/.gitignore

@@ -0,0 +1,5 @@
+.env
+.worktrees/
+.venv/
+__pycache__/
+*.pyc

{xplainable_preprocessing-0.2.3 → xplainable_preprocessing-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: xplainable-preprocessing
-Version: 0.2.3
+Version: 0.3.0
 Summary: Shared preprocessing pipeline package for xplainable
 Requires-Python: >=3.9
 Requires-Dist: cloudpickle>=3.0
@@ -8,6 +8,7 @@ Requires-Dist: numpy>=1.24
 Requires-Dist: pandas>=2.0
 Requires-Dist: pydantic>=2.0
 Requires-Dist: scikit-learn>=1.3
+Requires-Dist: scipy>=1.8
 Provides-Extra: dev
 Requires-Dist: pytest-cov; extra == 'dev'
 Requires-Dist: pytest>=7.0; extra == 'dev'

{xplainable_preprocessing-0.2.3 → xplainable_preprocessing-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "xplainable-preprocessing"
-version = "0.2.3"
+version = "0.3.0"
 description = "Shared preprocessing pipeline package for xplainable"
 requires-python = ">=3.9"
 dependencies = [
@@ -12,6 +12,7 @@ dependencies = [
     "scikit-learn>=1.3",
     "pandas>=2.0",
     "numpy>=1.24",
+    "scipy>=1.8",
     "cloudpickle>=3.0",
 ]
 

{xplainable_preprocessing-0.2.3 → xplainable_preprocessing-0.3.0}/src/xplainable_preprocessing/__init__.py

@@ -4,6 +4,14 @@ from xplainable_preprocessing.schema import validate_spec
 from xplainable_preprocessing.serialization import save_pipeline, load_pipeline
 from xplainable_preprocessing.pipeline import DataFramePipeline, DataFrameColumnTransformer
 from xplainable_preprocessing.registry import REGISTRY, register, generate_catalog
+from xplainable_preprocessing.guard import (
+    ROW_COLLAPSING,
+    preview_spec,
+    preview_step,
+    safe_catalog,
+    spec_safety_errors,
+    step_safety_error,
+)
 
 __all__ = [
     "PipelineSpec",
@@ -17,4 +25,10 @@ __all__ = [
     "REGISTRY",
     "register",
     "generate_catalog",
+    "ROW_COLLAPSING",
+    "safe_catalog",
+    "step_safety_error",
+    "spec_safety_errors",
+    "preview_step",
+    "preview_spec",
 ]

xplainable_preprocessing-0.3.0/src/xplainable_preprocessing/guard.py

@@ -0,0 +1,282 @@
+"""Step safety guard and step preview.
+
+Shared by the platform API (preprocessor create/add), the training service
+(agent apply/preview) and the client, so every surface rejects the same
+unsafe steps with the same message.
+
+A step is unsafe when it:
+- uses a row-collapsing transformer (one row per group; the column wrapper
+  concat-misaligns the result back onto the frame and corrupts other columns),
+- changes the row count,
+- drops the label column, or
+- mutates the label values.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any, Optional, Union
+
+import numpy as np
+import pandas as pd
+
+from xplainable_preprocessing.compiler import compile_spec
+from xplainable_preprocessing.pipeline import DataFrameColumnTransformer
+from xplainable_preprocessing.preview import compute_delta
+from xplainable_preprocessing.registry import generate_catalog
+from xplainable_preprocessing.schema import PipelineSpec, StepSpec
+
+ROW_COLLAPSING = frozenset({"GroupByAggTransformer"})
+"""Transformer types that change the row count by design. Never offer them to an LLM."""
+
+DEFAULT_MAX_ROWS = 10_000
+
+StepLike = Union[StepSpec, dict]
+SpecLike = Union[PipelineSpec, dict]
+
+
+def _as_step(step: StepLike) -> StepSpec:
+    return step if isinstance(step, StepSpec) else StepSpec(**step)
+
+
+def _as_spec(spec: SpecLike) -> PipelineSpec:
+    return spec if isinstance(spec, PipelineSpec) else PipelineSpec(**spec)
+
+
+def _sample(df: pd.DataFrame, max_rows: int) -> pd.DataFrame:
+    if max_rows and len(df) > max_rows:
+        return df.sample(max_rows, random_state=42)
+    return df.copy()
+
+
+def safe_catalog() -> str:
+    """The transformer catalog with row-collapsing entries removed."""
+    lines = generate_catalog().splitlines()
+    return "\n".join(
+        line for line in lines
+        if not any(line.lstrip().startswith(f"- {name}(") for name in ROW_COLLAPSING)
+    )
+
+
+def step_safety_error(
+    step: StepLike,
+    df: pd.DataFrame,
+    label: Optional[str] = None,
+    max_rows: int = DEFAULT_MAX_ROWS,
+) -> Optional[str]:
+    """Return why `step` is unsafe on `df`, or None when it is safe.
+
+    The step is compiled alone and fit on a sample. A step that cannot be
+    evaluated in isolation (for example it reads a column an earlier step
+    creates) is NOT rejected; use `spec_safety_errors` to validate a whole
+    spec in order.
+    """
+    step_spec = _as_step(step)
+    if step_spec.type in ROW_COLLAPSING:
+        return f"step '{step_spec.id}' uses row-collapsing transformer {step_spec.type}"
+    try:
+        sample = _sample(df, max_rows)
+        pipeline = compile_spec(PipelineSpec(steps=[step_spec]))
+        _, transformer = pipeline.steps[0]
+
+        # The column wrapper masks row collapses by concat-aligning onto the
+        # original index; check the inner transformer's raw output.
+        inner, inner_input = transformer, sample
+        if isinstance(transformer, DataFrameColumnTransformer):
+            inner = transformer.transformer
+            inner_input = sample[list(transformer.columns)]
+        raw = inner.fit_transform(inner_input.copy())
+        if hasattr(raw, "__len__") and len(raw) != len(sample):
+            return (
+                f"step '{step_spec.id}' changes the row count "
+                f"({len(sample)} -> {len(raw)}); row-collapsing steps corrupt the training frame"
+            )
+        transformed = pipeline.fit_transform(sample)
+    except Exception:
+        return None
+    return _label_error(step_spec.id, sample, transformed, label)
+
+
+def _label_error(step_id: str, before: pd.DataFrame, after: pd.DataFrame, label: Optional[str]) -> Optional[str]:
+    if not label:
+        return None
+    if label not in after.columns:
+        return f"step '{step_id}' drops the label column '{label}'"
+    b = before[label].reset_index(drop=True)
+    a = after[label].reset_index(drop=True)
+    if not isinstance(a, pd.Series) or not a.equals(b):
+        return f"step '{step_id}' mutates the label column '{label}'"
+    return None
+
+
+def spec_safety_errors(
+    spec: SpecLike,
+    df: pd.DataFrame,
+    label: Optional[str] = None,
+    max_rows: int = DEFAULT_MAX_ROWS,
+    report_failures: bool = True,
+) -> list:
+    """Validate every step of a spec in order on a sample of `df`.
+
+    Steps are applied cumulatively so later steps see the columns earlier
+    steps created. Returns a list of error strings (empty when the spec is
+    safe). A step that fails to fit is reported as well unless
+    `report_failures=False` (for callers that already surface fit failures
+    through their own compile path); such a step is skipped so later steps
+    are still checked.
+    """
+    pipeline_spec = _as_spec(spec)
+    errors = []
+    frame = _sample(df, max_rows)
+    for step_spec in pipeline_spec.steps:
+        if step_spec.type in ROW_COLLAPSING:
+            errors.append(f"step '{step_spec.id}' uses row-collapsing transformer {step_spec.type}")
+            continue
+        try:
+            single = compile_spec(PipelineSpec(steps=[step_spec]))
+            after = single.fit_transform(frame.copy())
+        except Exception as e:  # noqa: BLE001
+            if report_failures:
+                errors.append(f"step '{step_spec.id}' failed to apply: {e}")
+            continue
+        if len(after) != len(frame):
+            errors.append(
+                f"step '{step_spec.id}' changes the row count ({len(frame)} -> {len(after)})"
+            )
+            continue
+        label_err = _label_error(step_spec.id, frame, after, label)
+        if label_err:
+            errors.append(label_err)
+            continue
+        frame = after
+    return errors
+
+
+def _num(v: Any) -> Optional[float]:
+    try:
+        f = float(v)
+    except (TypeError, ValueError):
+        return None
+    return None if (math.isnan(f) or math.isinf(f)) else f
+
+
+def _column_stats(s: pd.Series) -> dict:
+    numeric = pd.api.types.is_numeric_dtype(s) and not pd.api.types.is_bool_dtype(s)
+    return {
+        "mean": _num(s.mean()) if numeric else None,
+        "nulls": int(s.isnull().sum()),
+        "min": _num(s.min()) if numeric else None,
+        "max": _num(s.max()) if numeric else None,
+        "unique": int(s.nunique()),
+    }
+
+
+def preview_step(
+    step: StepLike,
+    df: pd.DataFrame,
+    max_rows: int = DEFAULT_MAX_ROWS,
+) -> dict:
+    """Dry-run one step on a sample of `df`.
+
+    Returns
+    -------
+    dict with keys:
+        - delta: {dropped, added, updated} column lists
+        - stats: {column: {"before": stats, "after": stats}} for changed columns
+        - warnings: human-readable warnings (new nulls, infinities, row-count change, ...)
+        - rows_before / rows_after
+        - error: str when the step failed to apply, else None
+    """
+    step_spec = _as_step(step)
+    before = _sample(df, max_rows)
+    rows_before = len(before)
+    try:
+        pipeline = compile_spec(PipelineSpec(steps=[step_spec]))
+        after = pipeline.fit_transform(before.copy())
+    except Exception as e:  # noqa: BLE001
+        return {
+            "delta": {"dropped": [], "added": [], "updated": []},
+            "stats": {},
+            "warnings": [f"fit_transform failed: {e}"],
+            "rows_before": rows_before,
+            "rows_after": rows_before,
+            "error": str(e),
+        }
+
+    raw = compute_delta(before, after)
+    delta = {k: list(raw.get(k, [])) for k in ("dropped", "added", "updated")}
+
+    stats = {}
+    for col in delta["added"] + delta["updated"]:
+        entry = {}
+        if col in before.columns:
+            entry["before"] = _column_stats(before[col])
+        if col in after.columns:
+            entry["after"] = _column_stats(after[col])
+        if entry:
+            stats[col] = entry
+
+    warnings = []
+    for col in delta["updated"]:
+        if col in before.columns and col in after.columns:
+            new_nulls = int(after[col].isnull().sum()) - int(before[col].isnull().sum())
+            if new_nulls > 0:
+                warnings.append(f"Column '{col}': {new_nulls} new null values introduced")
+    for col in after.select_dtypes(include=["number"]).columns:
+        try:
+            inf_count = int(np.isinf(after[col].astype(float)).sum())
+        except (TypeError, ValueError):
+            inf_count = 0
+        if inf_count:
+            warnings.append(f"Column '{col}': {inf_count} infinite values detected")
+    if len(delta["added"]) > 100:
+        warnings.append(f"Massive column expansion: {len(delta['added'])} new columns added")
+    rows_after = len(after)
+    if rows_after != rows_before:
+        warnings.append(f"Row count changed from {rows_before} to {rows_after}")
+
+    return {
+        "delta": delta,
+        "stats": stats,
+        "warnings": warnings,
+        "rows_before": rows_before,
+        "rows_after": rows_after,
+        "error": None,
+    }
+
+
+def preview_spec(
+    spec: SpecLike,
+    df: pd.DataFrame,
+    label: Optional[str] = None,
+    max_rows: int = DEFAULT_MAX_ROWS,
+) -> dict:
+    """Preview every step of a spec in order, plus the safety verdict.
+
+    Returns
+    -------
+    dict with keys:
+        - steps: [{"step_id", "step_type", **preview_step(...)}] applied cumulatively
+        - safety_errors: from `spec_safety_errors`
+        - rows_before / rows_after
+        - output_columns: columns after the last step that applied cleanly
+    """
+    pipeline_spec = _as_spec(spec)
+    frame = _sample(df, max_rows)
+    rows_before = len(frame)
+    steps = []
+    for step_spec in pipeline_spec.steps:
+        result = preview_step(step_spec, frame, max_rows=0)
+        steps.append({"step_id": step_spec.id, "step_type": step_spec.type, **result})
+        if result["error"] is None:
+            try:
+                frame = compile_spec(PipelineSpec(steps=[step_spec])).fit_transform(frame.copy())
+            except Exception:  # noqa: BLE001
+                pass
+    return {
+        "steps": steps,
+        "safety_errors": spec_safety_errors(pipeline_spec, df, label=label, max_rows=max_rows),
+        "rows_before": rows_before,
+        "rows_after": len(frame),
+        "output_columns": list(frame.columns),
+    }

{xplainable_preprocessing-0.2.3 → xplainable_preprocessing-0.3.0}/src/xplainable_preprocessing/pipeline.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import pandas as pd
+from scipy import sparse
 from sklearn.base import BaseEstimator, TransformerMixin
 
 
@@ -20,6 +21,11 @@ class DataFrameColumnTransformer(BaseEstimator, TransformerMixin):
     def transform(self, X: pd.DataFrame) -> pd.DataFrame:
         Xt = X.copy()
         result = self.transformer.transform(Xt[self.columns])
+        if sparse.issparse(result):
+            # e.g. OneHotEncoder defaults to sparse_output=True; pd.DataFrame
+            # would treat the matrix as a single object column and raise
+            # "Shape of passed values is (N, 1), indices imply (N, K)".
+            result = result.toarray()
         if isinstance(result, pd.DataFrame):
             Xt = Xt.drop(columns=self.columns)
             Xt = pd.concat([Xt, result], axis=1)

xplainable_preprocessing-0.3.0/tests/test_guard.py

@@ -0,0 +1,119 @@
+"""Tests for the step safety guard and step preview."""
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from xplainable_preprocessing import (
+    ROW_COLLAPSING,
+    preview_spec,
+    preview_step,
+    safe_catalog,
+    spec_safety_errors,
+    step_safety_error,
+)
+from xplainable_preprocessing.schema import PipelineSpec, StepSpec
+
+
+@pytest.fixture
+def frame():
+    rng = np.random.default_rng(0)
+    n = 120
+    return pd.DataFrame({
+        "id": [f"C{i}" for i in range(n)],
+        "tenure": rng.integers(1, 72, n),
+        "charges": rng.uniform(20, 120, n).round(2),
+        "total": [str(v) for v in rng.uniform(100, 5000, n).round(1)],
+        "plan": rng.choice(["a", "b", "c", "d", "e"], n),
+        "churn": rng.integers(0, 2, n),
+    })
+
+
+DROP_ID = {"id": "drop_id", "type": "DropColumnsTransformer", "columns": ["id"]}
+CAST_TOTAL = {"id": "cast_total", "type": "TypeCastTransformer", "columns": ["total"], "params": {"dtypes": {"total": "float"}}}
+DROP_LABEL = {"id": "drop_label", "type": "DropColumnsTransformer", "columns": ["churn"]}
+COLLAPSE = {"id": "grp", "type": "GroupByAggTransformer", "columns": ["plan", "charges"], "params": {}}
+
+
+class TestSafeCatalog:
+    def test_excludes_row_collapsing_transformers(self):
+        catalog = safe_catalog()
+        for name in ROW_COLLAPSING:
+            assert f"- {name}(" not in catalog
+        assert "- DropColumnsTransformer(" in catalog
+
+
+class TestStepSafetyError:
+    def test_safe_step_is_none(self, frame):
+        assert step_safety_error(DROP_ID, frame, label="churn") is None
+        assert step_safety_error(StepSpec(**CAST_TOTAL), frame, label="churn") is None
+
+    def test_row_collapsing_type_is_rejected_without_running(self, frame):
+        assert "row-collapsing" in step_safety_error(COLLAPSE, frame)
+
+    def test_dropping_the_label_is_rejected(self, frame):
+        assert "drops the label" in step_safety_error(DROP_LABEL, frame, label="churn")
+
+    def test_mutating_the_label_is_rejected(self, frame):
+        step = {"id": "cast_label", "type": "TypeCastTransformer", "columns": ["churn"], "params": {"dtypes": {"churn": "str"}}}
+        assert "mutates the label" in step_safety_error(step, frame, label="churn")
+
+    def test_step_that_cannot_run_in_isolation_is_not_rejected(self, frame):
+        # References a column that does not exist yet (created by an earlier step).
+        step = {"id": "later", "type": "DropColumnsTransformer", "columns": ["derived_col"]}
+        assert step_safety_error(step, frame, label="churn") is None
+
+    def test_without_label_only_structural_checks_apply(self, frame):
+        assert step_safety_error(DROP_LABEL, frame) is None
+
+
+class TestSpecSafetyErrors:
+    def test_clean_spec_has_no_errors(self, frame):
+        spec = {"version": "2.0", "steps": [DROP_ID, CAST_TOTAL]}
+        assert spec_safety_errors(spec, frame, label="churn") == []
+
+    def test_reports_every_unsafe_step(self, frame):
+        spec = PipelineSpec(steps=[StepSpec(**DROP_ID), StepSpec(**DROP_LABEL), StepSpec(**COLLAPSE)])
+        errors = spec_safety_errors(spec, frame, label="churn")
+        assert len(errors) == 2
+        assert "drop_label" in errors[0] and "grp" in errors[1]
+
+    def test_steps_are_validated_cumulatively(self, frame):
+        # The second step reads a column the first step drops: in-order validation catches it.
+        spec = {"steps": [DROP_ID, {"id": "cast_id", "type": "TypeCastTransformer", "columns": ["id"], "params": {"dtypes": {"id": "str"}}}]}
+        errors = spec_safety_errors(spec, frame)
+        assert len(errors) == 1 and "cast_id" in errors[0] and "failed to apply" in errors[0]
+        assert spec_safety_errors(spec, frame, report_failures=False) == []
+
+
+class TestPreviewStep:
+    def test_reports_delta_stats_and_row_counts(self, frame):
+        out = preview_step(CAST_TOTAL, frame)
+        assert out["error"] is None
+        assert out["delta"]["updated"] == ["total"]
+        assert out["stats"]["total"]["after"]["mean"] is not None
+        assert out["stats"]["total"]["before"]["mean"] is None  # was text
+        assert out["rows_before"] == out["rows_after"] == len(frame)
+        assert out["warnings"] == []
+
+    def test_failure_is_reported_not_raised(self, frame):
+        bad = {"id": "bad", "type": "TypeCastTransformer", "columns": ["plan"], "params": {"dtypes": {"plan": "float"}, "errors": "raise"}}
+        out = preview_step(bad, frame)
+        assert out["error"] is not None or out["delta"]["updated"] == ["plan"]
+
+    def test_samples_large_frames(self, frame):
+        big = pd.concat([frame] * 100, ignore_index=True)
+        out = preview_step(DROP_ID, big, max_rows=500)
+        assert out["rows_before"] == 500
+
+
+class TestPreviewSpec:
+    def test_steps_apply_cumulatively_and_report_safety(self, frame):
+        spec = {"steps": [DROP_ID, CAST_TOTAL, DROP_LABEL]}
+        out = preview_spec(spec, frame, label="churn")
+        assert [s["step_id"] for s in out["steps"]] == ["drop_id", "cast_total", "drop_label"]
+        assert out["steps"][0]["delta"]["dropped"] == ["id"]
+        assert out["steps"][1]["delta"]["updated"] == ["total"]
+        assert "id" not in out["output_columns"]
+        assert out["safety_errors"] and "drop_label" in out["safety_errors"][0]
+        assert out["rows_before"] == out["rows_after"] == len(frame)

xplainable_preprocessing-0.3.0/tests/test_pipeline.py

@@ -0,0 +1,73 @@
+"""Tests for DataFrame-preserving pipeline pieces.
+
+Regression coverage for sparse transformer output: sklearn's OneHotEncoder
+defaults to sparse_output=True (csr_matrix). pd.DataFrame(csr_matrix, ...)
+treats the matrix as a single object column, raising
+"Shape of passed values is (N, 1), indices imply (N, K)".
+Observed live as HTTP 500s from /v1/preprocessors/create.
+"""
+
+import pandas as pd
+import pytest
+from sklearn.preprocessing import OneHotEncoder
+
+from xplainable_preprocessing.compiler import compile_spec
+from xplainable_preprocessing.pipeline import DataFrameColumnTransformer
+from xplainable_preprocessing.schema import PipelineSpec
+
+
+@pytest.fixture
+def df():
+    return pd.DataFrame({
+        "PhoneService": ["Yes", "No"] * 25,
+        "Contract": ["Month-to-month", "One year", "Two year", "One year",
+                     "Month-to-month"] * 10,
+        "tenure": range(50),
+    })
+
+
+class TestSparseTransformerOutput:
+    def test_column_transformer_densifies_sparse_output(self, df):
+        t = DataFrameColumnTransformer(OneHotEncoder(), ["PhoneService", "Contract"])
+        out = t.fit_transform(df)
+
+        assert isinstance(out, pd.DataFrame)
+        assert "tenure" in out.columns
+        assert "PhoneService_Yes" in out.columns
+        assert "Contract_Two year" in out.columns
+        # Row 0: PhoneService == "Yes"
+        assert out.loc[0, "PhoneService_Yes"] == 1.0
+        assert out.loc[0, "PhoneService_No"] == 0.0
+
+    def test_compile_spec_onehot_multi_column_step(self, df):
+        spec = PipelineSpec(version="2.0", steps=[
+            {"id": "oh", "type": "OneHotEncoder",
+             "columns": ["PhoneService", "Contract"], "params": {}},
+        ])
+        pipeline = compile_spec(spec)
+        out = pipeline.fit_transform(df)
+
+        assert "PhoneService" not in out.columns
+        assert "PhoneService_No" in out.columns
+        assert "Contract_One year" in out.columns
+        assert len(out) == 50
+
+    def test_compile_spec_onehot_one_column_per_step(self, df):
+        spec = PipelineSpec(version="2.0", steps=[
+            {"id": "oh1", "type": "OneHotEncoder",
+             "columns": ["PhoneService"], "params": {}},
+            {"id": "oh2", "type": "OneHotEncoder",
+             "columns": ["Contract"], "params": {}},
+        ])
+        pipeline = compile_spec(spec)
+        out = pipeline.fit_transform(df)
+
+        assert "PhoneService_Yes" in out.columns
+        assert "Contract_Month-to-month" in out.columns
+        assert "tenure" in out.columns
+
+    def test_transform_after_fit_matches_fit_transform(self, df):
+        t = DataFrameColumnTransformer(OneHotEncoder(), ["PhoneService"])
+        fitted_out = t.fit_transform(df)
+        transform_out = t.transform(df)
+        pd.testing.assert_frame_equal(fitted_out, transform_out)

xplainable_preprocessing-0.2.3/.gitignore

@@ -1 +0,0 @@
-.env