xplainable-preprocessing 0.3.1__tar.gz → 0.4.0__tar.gz

xplainable_preprocessing-0.4.0/.github/workflows/ci.yml

@@ -0,0 +1,90 @@
+name: CI
+
+# Tests on both Pythons the platform runs (core-api and the inference server
+# on 3.10, the train service on 3.12), plus a cross-version check that a
+# pipeline with custom steps fitted and saved under one Python loads and
+# transforms identically under the other.
+
+on:
+  push:
+    branches: [main, 'release/**']
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: pytest (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # core-api / inference-server stack
+          - python-version: '3.10'
+            pins: 'pandas==2.3.3 numpy==2.0.2 scikit-learn==1.6.1'
+          # train-service stack
+          - python-version: '3.12'
+            pins: 'pandas==2.3.3'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: pip install -e '.[dev]' ${{ matrix.pins }}
+      - name: Test
+        run: python -m pytest -q
+
+  portability-dump:
+    name: dump under Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - python-version: '3.10'
+            pins: 'pandas==2.3.3 numpy==2.0.2 scikit-learn==1.6.1'
+          - python-version: '3.12'
+            pins: 'pandas==2.3.3'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: pip install -e . ${{ matrix.pins }}
+      - name: Fit and save the bike_sharing custom-step pipeline
+        run: python tests/portability_roundtrip.py dump dump
+      - uses: actions/upload-artifact@v4
+        with:
+          name: pipeline-py${{ matrix.python-version }}
+          path: dump/
+
+  portability-load:
+    name: load a Python ${{ matrix.written-by }} binary under Python ${{ matrix.python-version }}
+    needs: portability-dump
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - python-version: '3.12'
+            written-by: '3.10'
+            pins: 'pandas==2.3.3'
+          - python-version: '3.10'
+            written-by: '3.12'
+            pins: 'pandas==2.3.3 numpy==2.0.2 scikit-learn==1.6.1'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: pip install -e . ${{ matrix.pins }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: pipeline-py${{ matrix.written-by }}
+          path: dump/
+      - name: Load, transform and compare
+        run: python tests/portability_roundtrip.py load dump

{xplainable_preprocessing-0.3.1 → xplainable_preprocessing-0.4.0}/.gitignore

@@ -3,3 +3,5 @@
 .venv/
 __pycache__/
 *.pyc
+dist/
+build/

xplainable_preprocessing-0.4.0/CHANGELOG.md

@@ -0,0 +1,231 @@
+# Changelog
+
+All notable changes to xplainable-preprocessing. Versions follow semantic
+versioning; before 1.0 a minor release may break compatibility, and every
+break is listed under **Breaking changes**.
+
+## [0.4.0] - 2026-09-28
+
+A breaking release. Pipelines now refuse to produce two columns with the same
+name, expressions run on an allow-listed evaluator instead of `pandas.eval`,
+several transformers keep data they used to destroy, and custom steps pickle
+so a binary loads on another Python version. It also adds seven transformers
+and four modules: `detect`, `io`, `numeric_text` and `contracts`.
+
+Supported: Python 3.9 and later (tested on 3.10 and 3.12), pandas 2.x
+(`pandas>=2.0,<3`), numpy 1.24 or later, scikit-learn 1.3 or later.
+
+### Breaking changes
+
+| Area | Change |
+|---|---|
+| Column collisions | `DataFrameColumnTransformer` raises `ColumnCollisionError` when a wrapped step emits a column that is also a pass-through column. `DataFramePipeline` raises it, with the step id, when any step's output repeats a column name (for example `RenameColumnsTransformer` renaming onto an existing column). 0.3.x returned a frame with two columns of that name. The error carries `columns`, `step` and `inputs`, and pickles intact. |
+| Expressions | `ExpressionTransformer` parses the expression with `ast` and evaluates it with an allow-list: arithmetic, comparisons, element-wise logic, the `pandas.eval` math functions, column references (bare, backticked, `df['x']`, `df.x`), and the methods `.where .abs .round .isna .notna .isnull .notnull .fillna .clip .between .isin .astype` (restricted dtypes) plus `.str.len/count/contains/startswith/endswith/lower/upper/strip`. Anything else (other methods such as `.cumsum()`, `.shift()`, `.rank()` or `.pow()`, dunders, I/O) raises `ExpressionError` at fit and at transform. Nothing reaches `eval` or `pandas.eval`. A column named `df` is read as `` `df` `` or `df['df']`. |
+| `save_pipeline` | Defaults to `portable=True` and raises `NonPortablePipelineError`, naming the steps, when any fitted state would be pickled by value (a lambda, or a function or class from custom code kept on `self`). `portable=False` saves as 0.3.x did. |
+| TextCleanTransformer | Operations apply to `str` cells only. Numbers, None and other values in a mixed column are kept; 0.3.x turned them into NaN. `string` and `category` columns are cleaned too (a category column stays categorical). An unknown operation raises at fit; 0.3.x ignored it. |
+| GroupedLagTransformer, RollingAggTransformer | Compute on a stably sorted copy and return rows in input order with the input index. With `order_by`, 0.3.x returned the rows sorted by group and `order_by`. Each row's values are unchanged, but row order is not. Negative lag periods and missing `group_by`/`order_by` columns raise at fit. Categorical group keys use `observed=True`. |
+| DateTimeExtractTransformer without step columns | At fit it picks datetime columns, and text columns where at least 80% of values parse as dates. Only those are parsed and dropped. Numeric, bool and other text columns are left untouched; 0.3.x converted every column to dates, extracted parts and dropped it. Fit raises when no column holds dates. |
+| ClipTransformer | Bounds are validated at fit: a non-number, `min > max`, or a `bounds` column that is missing or not numeric raises. |
+| `generate_catalog()` | Lines show typed params (`name: type = default`). A required param is shown without a default, and a choice between params is spelled out ("Requires order or mapping."). Code that parses catalog lines must follow the new format. |
+| Guard and previews | `step_safety_error` and `spec_safety_errors` also reject steps that reorder rows, read the label, or copy the label (see Added). Checks and previews run on the whole frame up to `full_rows_limit` (100k) rows, where 0.3.x used a random 10k-row sample. They see every row, and take longer on large frames. |
+
+#### Stored 0.3.x pipelines
+
+Built-in steps pickle by reference, so a 0.3.x binary loads into the 0.4.0
+classes. The 0.3.x pickles of `RollingAggTransformer`, `ClipTransformer` and
+`DateTimeExtractTransformer` are filled in with the new params' defaults. A
+0.3.x pipeline of built-in steps serves the same output under 0.4.0
+(`tests/test_legacy_pickles.py` checks this on binaries written by 0.3.1)
+with these exceptions:
+
+| Stored pipeline | Under 0.4.0 | What to do |
+|---|---|---|
+| A wrapped step whose output name already exists, or a rename onto an existing column | `transform` raises `ColumnCollisionError` (0.3.x served duplicate columns) | Re-fit with a new output name, or drop the existing column first |
+| An expression outside the allow-list | `transform` raises `ExpressionError` | Rewrite it with allowed constructs (`a ** 2` for `a.pow(2)`) and re-fit, or ask for the method to be allow-listed |
+| TextCleanTransformer on a column mixing text and other values | Non-text cells are kept, so the output changes | Re-fit, or accept the change |
+| GroupedLagTransformer or RollingAggTransformer with `order_by` | Rows come back in input order | Re-fit, and check anything that relied on the sorted order |
+| DateTimeExtractTransformer without step columns | Non-date columns are kept, so the output columns change. A 0.3.x binary records no date columns, so each served batch decides which columns are dates: a single row with an empty date keeps that column and gets no date parts | Re-fit |
+| Custom steps saved by 0.3.x (pickled by value) | Load and serve on the Python version that wrote them, as before; on another Python version `load_pipeline` still fails (typically a `TypeError` from the pickled code). `save_pipeline` refuses them unless `portable=False` | Re-fit to get a portable binary |
+
+Steps from scikit-learn (`OneHotEncoder`, `StandardScaler`, `SimpleImputer`,
+...) pickle as scikit-learn's own objects. They load only under the
+scikit-learn release that fitted them (a 1.6.1 `SimpleImputer` fails under
+1.9), whichever xplainable-preprocessing release is installed.
+
+### Added
+
+**Transformers** (registered, in the catalog, with required params in
+`REQUIRED_PARAMS`):
+- `ParseNumericTransformer`: numbers stored as text into floats. Handles `%`,
+  currency, `1,234`, `(1,234)`, units such as `36 months`, ranges, bounds
+  such as `10+` and NA tokens. Idempotent on numbers.
+- `NullTokensTransformer`: placeholder text (`?`, `N/A`, blanks) and sentinel
+  numbers (`999`) to NaN, with optional 0/1 flags.
+- `OrdinalMapTransformer`: an ordered scale to numbers in place, by `order`
+  or `mapping`. NaN- and unknown-safe, with JSON-safe params. It rejects a
+  mapping that gives one level two codes.
+- `CrossCategoriesTransformer`: 2 to 3 low-cardinality columns crossed into
+  one readable category, with zero padding and a `max_levels` cap.
+- `TimeLagTransformer`: lags and trailing-window aggregates at exact time
+  offsets over strictly earlier timestamps, per group. Supports `gap`,
+  deduplication of repeated timestamps, a bounded history tail and
+  `serving_requirements()`. A `gap` that covers every lag is the one safe way
+  to use past target values.
+- `GroupBroadcastTransformer`: a group aggregate written to every row of its
+  group. A group can be a calendar date, hour, week or month of a time
+  column, and the mapping is kept from fit so single-row serving works.
+- `TextFeaturesTransformer`: free text into numeric features (length, words,
+  digits, capitals, punctuation, URLs, emails, phone-like numbers, whole-word
+  keyword flags). The feature set is frozen as `feature_set_version=1`.
+
+**Transformer options:**
+- `RollingAggTransformer`: `closed` (`"left"` excludes the current row) and
+  `shift`. List params accept a single name.
+- `GroupedLagTransformer`: `order_by` takes a list, and scalar params are
+  coerced.
+- `DateTimeExtractTransformer`: components `elapsed_days`, `elapsed_hours`
+  (from an `origin` fixed at fit), `hour_of_week`, `date` and `month_index`,
+  plus `as_categorical` (zero-padded strings). The text format is fixed at
+  fit, and ISO 8601 values it misses are still read.
+- `ClipTransformer`: per-column `bounds` (`{"x": [3, 11], "z": [1.5, None]}`).
+- `TextCleanTransformer`: operations `html_unescape`, `fix_encoding` (cp1252
+  mojibake), `nfkc`, `mask_digits`, `mask_urls` and `mask_emails`.
+
+**Modules:**
+- `numeric_text`: `parse_numeric` and `numeric_text_report`, which says
+  whether a text column holds numbers. The report rejects dates, codes with
+  leading zeros and identifiers, and reports collisions and mixed units. The
+  module also holds the NA-token vocabulary (`NA_TOKENS`, `MAYBE_NA_TOKENS`).
+- `io.read_table(content, filename, options) -> (DataFrame, ReadReport)`:
+  one reader for uploaded CSV and Excel files.
+  - Encoding: UTF-8 with cp1252 and latin-1 fallbacks.
+  - Delimiter sniffing, with a guard against single-column reads.
+  - Header rows below the top and headerless files are detected from the
+    raw bytes and read again losslessly.
+  - NA policies: `literal_v1` (the default; only empty cells are missing,
+    and text tokens stay literal) and `legacy` (pandas' defaults).
+  - Typed Excel with a safe 99% numeric coercion.
+  - `ReadError` subclasses carry messages meant for users.
+  - `coerce_numeric_columns` applies the same coercion to frames already
+    stored.
+  - Excel needs the new `excel` extra: `pip install
+    "xplainable-preprocessing[excel]"`.
+- `detect`: pure functions that describe a raw frame and return JSON-safe
+  `Issue`s. Each issue has a severity, a confidence and, where a library
+  step fixes it, `suggested_steps`.
+  - `scan_frame`: column content. Placeholder and maybe-NA tokens, numbers
+    stored as text, sentinels and top or bottom codes, structural and
+    co-missing gaps, whitespace and case variants, impossible years and
+    broken year orderings, ordinal scales, free text, sparse markers,
+    impossible zeros, and a regression target's point mass.
+  - `scan_structure`: frame shape. A delimited file read as one column,
+    headerless files (pandas' `.N` suffixes undone), header rows inside the
+    data, repeated headers, junk rows, and identifiers measured against
+    distinct rows.
+  - `scan_relations`: needs a target. Target proxies, columns that rebuild
+    the target, time-order drift, duplicate keys whose target is constant,
+    and 1:1 or functional redundancy.
+  - `scan_all` runs all three.
+  - `repair_frame` applies only lossless repairs to frames already stored.
+- `contracts`:
+  - `dependency_map(fitted, spec, data)`: for each engineered output column,
+    the step that computed it, its inputs and raw sources, and whether it
+    can be recomputed row by row. For ExpressionTransformer outputs it also
+    gives a `DataFrame.eval` formula, verified on the rows, that
+    xplainable-gm's optimiser accepts as a `derived` declaration.
+    `DependencyMap.is_engineered(column)` checks a single column.
+  - `partial_transform(fitted, base_prepared, raw_rows, changed_columns)`:
+    recomputes only the outputs that a changed raw column feeds.
+  - Both work on 0.3.x pipelines when the spec is passed.
+
+**Pipelines and serialisation:**
+- `compile_spec` attaches the spec to the pipeline as `pipeline.spec`, as a
+  plain dict.
+- Custom steps pickle as their spec plus fitted state. On load, the
+  module-level `sandbox.rebuild_custom` validates and runs the source again,
+  so a pipeline saved on Python 3.10 loads on 3.12 and the reverse. Copies
+  keep the original class.
+- A custom step's constructor `TypeError` names the params its `__init__`
+  does not accept.
+
+**Guard and previews** (`guard`, `preview`):
+- `step_safety_error` and `spec_safety_errors` reject four more kinds of
+  step:
+  - a step that reorders rows;
+  - a step that reads the label. This is checked statically (columns,
+    params, expression names, custom-code AST) and behaviourally (transform
+    with the label dropped or shuffled). A `TimeLagTransformer` whose `gap`
+    covers every lag is exempt (`LABEL_READ_EXEMPTIONS`).
+  - a step that adds a copy of the label: the new column equals the label
+    on at least 50% of rows, or has |Spearman rho| >= 0.995. A preview
+    warns from 5% up.
+  - a step whose output collides with an existing column.
+- `preview_step` and `preview_spec` add:
+  - a `label` parameter;
+  - `serving_warnings`, from a single-row batch-dependence probe;
+  - `changes`, with `changed_cells` and `top_changes` for each updated
+    column;
+  - `top_values` for non-numeric columns;
+  - a warning when a step changes 0 cells;
+  - `rows_total` and `sampled`.
+- Past 100k rows, checks and previews use an order-preserving sample. Lag,
+  rolling, group-statistic and custom steps get a contiguous block; any
+  other step gets rows spread over the frame that always include the
+  extreme rows of its columns.
+- `registry.missing_required_params` and `REQUIRED_PARAMS` list what each
+  step type must set. `registry.catalog_entry` builds one catalog line.
+
+**Package:**
+- `__version__`. The top level now exports `ColumnCollisionError`,
+  `NonPortablePipelineError`, `ExpressionError`, `read_table`, `ReadReport`,
+  `ReadError`, `Issue`, `scan_all`, `parse_numeric` and
+  `numeric_text_report`. The `detect`, `io` and `numeric_text` submodules are
+  reachable as attributes.
+- CI runs the suite on Python 3.10 (pandas 2.3.3, numpy 2.0.2,
+  scikit-learn 1.6.1) and 3.12, plus a matrix that dumps a pipeline on 3.10
+  and loads it on 3.12, and the reverse.
+
+### Fixed
+
+- `infer_schema`, `compute_preview` and `compute_step_deltas` returned samples
+  that `json.dumps` rejects when a column held timestamps, timedeltas, periods,
+  numpy scalars or pandas NA values. Storing such a schema failed, e.g. after a
+  `DateTimeExtractTransformer` with `drop_original=False`. Samples are now
+  JSON-safe: missing values become null, timestamps and dates ISO-8601 text,
+  timedeltas ISO-8601 durations, numpy scalars Python values.
+- A step whose output repeated a column name produced a frame with two
+  columns of that name, and anything downstream then failed or read the
+  wrong one. It now raises (see Breaking changes).
+- `GroupedLagTransformer` and `RollingAggTransformer` misaligned rows when
+  `order_by` sorted them. `RollingAggTransformer` crashed on categorical
+  group keys with unused categories.
+- Unwrapped `DateTimeExtractTransformer` destroyed every non-date column.
+- `TextCleanTransformer` erased numbers in mixed columns.
+- Custom-step pipelines saved on one Python version failed to load on
+  another (`code() argument 13 must be str, not int`).
+- `preview_step` no longer raises when statistics or deltas fail, and
+  repeated input column names come back as an error.
+
+### Known limitations
+
+- pandas 3 is not supported, and the dependency is capped at `<3`. Under
+  pandas 3.0 the suite fails in four places:
+  - `detect.target` writes into arrays that copy-on-write makes read-only,
+    so most relation scans raise.
+  - `detect.structure`'s headerless restore handles missing values in the
+    default `str` dtype differently.
+  - `ExpressionTransformer`'s `.str` methods also handle missing values in
+    the `str` dtype differently.
+  - `CategoryCondenseTransformer` fits only `object` and `category`
+    columns, so it leaves `str` columns uncondensed.
+
+## [0.3.1]
+
+- `ExpressionTransformer` resolves backtick-quoted column names.
+
+## [0.3.0]
+
+- Step safety guard (`step_safety_error`, `spec_safety_errors`), step and
+  spec previews (`preview_step`, `preview_spec`), and `safe_catalog`.
+
+Earlier releases are described in the git history.

xplainable_preprocessing-0.4.0/PKG-INFO

@@ -0,0 +1,200 @@
+Metadata-Version: 2.5
+Name: xplainable-preprocessing
+Version: 0.4.0
+Summary: Shared preprocessing pipeline package for xplainable
+Requires-Python: >=3.9
+Requires-Dist: cloudpickle>=3.0
+Requires-Dist: numpy>=1.24
+Requires-Dist: pandas<3,>=2.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: scikit-learn>=1.3
+Requires-Dist: scipy>=1.8
+Provides-Extra: dev
+Requires-Dist: openpyxl>=3.1; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: xlrd>=2.0.1; extra == 'dev'
+Provides-Extra: excel
+Requires-Dist: openpyxl>=3.1; extra == 'excel'
+Requires-Dist: xlrd>=2.0.1; extra == 'excel'
+Description-Content-Type: text/markdown
+
+# xplainable-preprocessing
+
+Shared preprocessing pipeline library for [xplainable](https://xplainable.io). Define, compile, fit, serialise, and apply data transformation pipelines using a declarative JSON spec.
+
+## Install
+
+```bash
+pip install xplainable-preprocessing
+pip install "xplainable-preprocessing[excel]"   # io.read_table for .xlsx/.xls
+```
+
+Upgrading from 0.3.x? 0.4.0 has breaking changes; see [CHANGELOG.md](CHANGELOG.md).
+
+## Quick Start
+
+### Define a pipeline as JSON
+
+```python
+from xplainable_preprocessing.schema import PipelineSpec
+from xplainable_preprocessing.compiler import compile_spec
+
+spec = PipelineSpec(
+    version="2.0",
+    steps=[
+        {"id": "drop_ids", "type": "DropColumnsTransformer", "params": {"columns": ["customer_id", "row_id"]}},
+        {"id": "fill_numeric", "type": "FillMissingTransformer", "columns": ["tenure", "charges"], "params": {"strategies": {"tenure": "median", "charges": "median"}}},
+        {"id": "clip_outliers", "type": "ClipTransformer", "columns": ["charges"], "params": {"min_val": 0, "max_val": 500}},
+        {"id": "missing_flags", "type": "MissingFlagTransformer", "columns": ["tenure", "charges"]},
+        {"id": "condense_plan", "type": "CategoryCondenseTransformer", "columns": ["plan_type"], "params": {"max_categories": 10}},
+        {"id": "extract_dates", "type": "DateTimeExtractTransformer", "columns": ["signup_date"], "params": {"components": ["month", "dayofweek"], "drop_original": true}},
+    ]
+)
+
+pipeline = compile_spec(spec)
+```
+
+### Fit and transform
+
+```python
+import pandas as pd
+
+df = pd.read_csv("data.csv")
+pipeline.fit(df)
+transformed = pipeline.transform(df)
+```
+
+### Serialise and load
+
+```python
+from xplainable_preprocessing.serialization import save_pipeline, load_pipeline
+
+# Save
+binary = save_pipeline(pipeline)
+
+# Load
+loaded = load_pipeline(binary)
+result = loaded.transform(new_data)
+```
+
+## Available Transformers
+
+### Data Cleaning
+| Transformer | Description |
+|-------------|-------------|
+| `DropColumnsTransformer` | Drop specified columns |
+| `RenameColumnsTransformer` | Rename columns |
+| `TypeCastTransformer` | Cast column dtypes (e.g. str to float64, with error coercion) |
+| `ClipTransformer` | Set numeric values outside (min, max), or per-column `bounds`, to NaN |
+| `ParseNumericTransformer` | Parse numbers stored as text (%, currency, `1,234`, `36 months`, ranges, `10+`) |
+| `NullTokensTransformer` | Turn placeholder text (`?`, `N/A`) and sentinel numbers (`999`) into NaN |
+
+### Missing Values
+| Transformer | Description |
+|-------------|-------------|
+| `FillMissingTransformer` | Fill nulls with median, mode, constant, or other strategies |
+| `MissingFlagTransformer` | Create binary 0/1 indicator columns for nulls |
+
+### Feature Engineering
+| Transformer | Description |
+|-------------|-------------|
+| `ExpressionTransformer` | Create derived columns from expressions (allow-listed evaluator) |
+| `DateTimeExtractTransformer` | Extract year, month, dayofweek, elapsed time, etc. from datetime columns |
+| `GroupByAggTransformer` | Aggregate features by group (count, sum, mean) |
+| `GroupedLagTransformer` | Create row-shift lags within groups |
+| `RollingAggTransformer` | Rolling window aggregations |
+| `TimeLagTransformer` | Lags and trailing windows at exact time offsets over strictly earlier rows |
+| `GroupBroadcastTransformer` | Write a group aggregate (e.g. a holiday flag per date) to every row of the group |
+| `TextFeaturesTransformer` | Numeric features from free text (length, digits, URLs, keyword flags) |
+
+### Categorical
+| Transformer | Description |
+|-------------|-------------|
+| `CategoryCondenseTransformer` | Condense high-cardinality categoricals to top N + "Other" |
+| `TextCleanTransformer` | Clean text cells: lowercase, strip, remove HTML/extra whitespace, fix encoding, mask digits/URLs/emails |
+| `OrdinalMapTransformer` | Map an ordered scale (grades, ratings, bands) to numbers |
+| `CrossCategoriesTransformer` | Cross 2-3 low-cardinality columns into one category |
+
+### Scaling (use with caution -- breaks xplainable model explainability)
+| Transformer | Description |
+|-------------|-------------|
+| `StandardScaler` | Standardise to zero mean, unit variance |
+| `MinMaxScaler` | Scale to [0, 1] range |
+| `RobustScaler` | Scale using median and IQR |
+| `PowerTransformer` | Apply power transform for normality |
+| `QuantileTransformer` | Transform to uniform or normal distribution |
+
+> **Note:** Scaling transformers are available but should NOT be used with xplainable models. xplainable models are inherently explainable and require raw feature values. Scaling destroys interpretability.
+
+### Other sklearn
+| Transformer | Description |
+|-------------|-------------|
+| `SimpleImputer` | sklearn's imputer |
+| `OneHotEncoder` | One-hot encode categoricals |
+| `OrdinalEncoder` | Ordinal encode categoricals |
+| `KBinsDiscretizer` | Discretise continuous features into bins |
+| `Binarizer` | Threshold features to binary |
+
+## Pipeline Spec Format
+
+Pipelines are defined as JSON for portability across the xplainable platform, MCP tools, and API:
+
+```json
+{
+    "version": "2.0",
+    "steps": [
+        {
+            "id": "step_name",
+            "type": "TransformerName",
+            "columns": ["col1", "col2"],
+            "params": {"param1": "value1"},
+            "description": "Optional description"
+        }
+    ]
+}
+```
+
+- `id`: unique step identifier
+- `type`: transformer class name (from the registry)
+- `columns`: optional list of columns to apply to (wraps in `DataFrameColumnTransformer`)
+- `params`: constructor arguments for the transformer
+- `description`: optional human-readable description
+
+## Architecture
+
+```
+PipelineSpec (JSON)
+    ↓ compile_spec()
+DataFramePipeline (sklearn Pipeline of transformers)
+    ↓ fit() / transform()
+Transformed DataFrame
+    ↓ save_pipeline()
+Binary (cloudpickle)
+    ↓ load_pipeline()
+DataFramePipeline (restored)
+```
+
+Key modules:
+- `schema.py` -- Pydantic models for `PipelineSpec` and `StepSpec`
+- `compiler.py` -- compiles JSON spec into sklearn pipeline
+- `pipeline.py` -- `DataFramePipeline` wrapper around sklearn `Pipeline`
+- `registry.py` -- maps transformer names to classes, generates LLM-ready catalog
+- `serialization.py` -- cloudpickle save/load; portable across Python versions
+- `guard.py` -- step safety checks and step/spec previews
+- `preview.py` -- before/after preview with schema deltas
+- `contracts.py` -- which engineered columns each step computes (`dependency_map`, `partial_transform`)
+- `detect/` -- detectors that describe a raw frame (`scan_all`, `Issue`)
+- `io.py` -- `read_table`, the shared CSV/Excel reader
+- `numeric_text.py` -- parse numbers stored as text
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+MIT

{xplainable_preprocessing-0.3.1 → xplainable_preprocessing-0.4.0}/README.md

@@ -6,8 +6,11 @@ Shared preprocessing pipeline library for [xplainable](https://xplainable.io). D
 
 ```bash
 pip install xplainable-preprocessing
+pip install "xplainable-preprocessing[excel]"   # io.read_table for .xlsx/.xls
 ```
 
+Upgrading from 0.3.x? 0.4.0 has breaking changes; see [CHANGELOG.md](CHANGELOG.md).
+
 ## Quick Start
 
 ### Define a pipeline as JSON
@@ -62,7 +65,9 @@ result = loaded.transform(new_data)
 | `DropColumnsTransformer` | Drop specified columns |
 | `RenameColumnsTransformer` | Rename columns |
 | `TypeCastTransformer` | Cast column dtypes (e.g. str to float64, with error coercion) |
-| `ClipTransformer` | Clip numeric values outside (min, max) to NaN |
+| `ClipTransformer` | Set numeric values outside (min, max), or per-column `bounds`, to NaN |
+| `ParseNumericTransformer` | Parse numbers stored as text (%, currency, `1,234`, `36 months`, ranges, `10+`) |
+| `NullTokensTransformer` | Turn placeholder text (`?`, `N/A`) and sentinel numbers (`999`) into NaN |
 
 ### Missing Values
 | Transformer | Description |
@@ -73,17 +78,22 @@ result = loaded.transform(new_data)
 ### Feature Engineering
 | Transformer | Description |
 |-------------|-------------|
-| `ExpressionTransformer` | Create derived columns via pandas expressions |
-| `DateTimeExtractTransformer` | Extract year, month, dayofweek, etc. from datetime columns |
+| `ExpressionTransformer` | Create derived columns from expressions (allow-listed evaluator) |
+| `DateTimeExtractTransformer` | Extract year, month, dayofweek, elapsed time, etc. from datetime columns |
 | `GroupByAggTransformer` | Aggregate features by group (count, sum, mean) |
-| `GroupedLagTransformer` | Create lagged features within groups |
+| `GroupedLagTransformer` | Create row-shift lags within groups |
 | `RollingAggTransformer` | Rolling window aggregations |
+| `TimeLagTransformer` | Lags and trailing windows at exact time offsets over strictly earlier rows |
+| `GroupBroadcastTransformer` | Write a group aggregate (e.g. a holiday flag per date) to every row of the group |
+| `TextFeaturesTransformer` | Numeric features from free text (length, digits, URLs, keyword flags) |
 
 ### Categorical
 | Transformer | Description |
 |-------------|-------------|
 | `CategoryCondenseTransformer` | Condense high-cardinality categoricals to top N + "Other" |
-| `TextCleanTransformer` | Clean text: lowercase, strip, remove HTML/extra whitespace |
+| `TextCleanTransformer` | Clean text cells: lowercase, strip, remove HTML/extra whitespace, fix encoding, mask digits/URLs/emails |
+| `OrdinalMapTransformer` | Map an ordered scale (grades, ratings, bands) to numbers |
+| `CrossCategoriesTransformer` | Cross 2-3 low-cardinality columns into one category |
 
 ### Scaling (use with caution -- breaks xplainable model explainability)
 | Transformer | Description |
@@ -149,8 +159,13 @@ Key modules:
 - `compiler.py` -- compiles JSON spec into sklearn pipeline
 - `pipeline.py` -- `DataFramePipeline` wrapper around sklearn `Pipeline`
 - `registry.py` -- maps transformer names to classes, generates LLM-ready catalog
-- `serialization.py` -- cloudpickle save/load
+- `serialization.py` -- cloudpickle save/load; portable across Python versions
+- `guard.py` -- step safety checks and step/spec previews
 - `preview.py` -- before/after preview with schema deltas
+- `contracts.py` -- which engineered columns each step computes (`dependency_map`, `partial_transform`)
+- `detect/` -- detectors that describe a raw frame (`scan_all`, `Issue`)
+- `io.py` -- `read_table`, the shared CSV/Excel reader
+- `numeric_text.py` -- parse numbers stored as text
 
 ## Development
 

{xplainable_preprocessing-0.3.1 → xplainable_preprocessing-0.4.0}/pyproject.toml

@@ -4,22 +4,33 @@ build-backend = "hatchling.build"
 
 [project]
 name = "xplainable-preprocessing"
-version = "0.3.1"
+version = "0.4.0"
 description = "Shared preprocessing pipeline package for xplainable"
+readme = "README.md"
 requires-python = ">=3.9"
 dependencies = [
     "pydantic>=2.0",
     "scikit-learn>=1.3",
-    "pandas>=2.0",
+    # pandas 3 is not supported yet: its copy-on-write read-only arrays and
+    # default string dtype break detect.target, detect.structure and a few
+    # transformers (see CHANGELOG 0.4.0).
+    "pandas>=2.0,<3",
     "numpy>=1.24",
     "scipy>=1.8",
     "cloudpickle>=3.0",
 ]
 
 [project.optional-dependencies]
+# Excel uploads in xplainable_preprocessing.io.read_table: .xlsx and .xls.
+excel = [
+    "openpyxl>=3.1",
+    "xlrd>=2.0.1",
+]
 dev = [
     "pytest>=7.0",
     "pytest-cov",
+    "openpyxl>=3.1",
+    "xlrd>=2.0.1",
 ]
 
 [tool.hatch.build.targets.wheel]