universal-data-cleaner 2.1.0__tar.gz → 2.3.0__tar.gz

{universal_data_cleaner-2.1.0 → universal_data_cleaner-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: universal-data-cleaner
-Version: 2.1.0
+Version: 2.3.0
 Summary: Production-ready, scikit-learn-compatible data cleaning transformer for pandas.
 Author: Haseeb Khan
 License-Expression: MIT
@@ -153,9 +153,9 @@ For a column that should be numeric even though it arrived as dirty strings:
 
 ```python
 config = CleanerConfig(
-    numeric_columns=("salary",),          # force numeric parsing
-    datetime_columns=("joined",),        # parse dates instead of treating them as categories
-    unseen_category="keep",              # or "fill" to replace unseen labels
+    numeric_columns=("salary",),  # force numeric parsing
+    datetime_columns=("joined",),  # parse dates instead of treating them as categories
+    unseen_category="keep",  # or "fill" to replace unseen labels
 )
 ```
 
@@ -171,7 +171,7 @@ X_train, X_test, y_train, y_test = train_test_split(
 cleaner = DataCleaner(config).fit(X_train)
 
 X_train_clean = cleaner.transform(X_train)
-X_test_clean = cleaner.transform(X_test)   # uses train statistics only
+X_test_clean = cleaner.transform(X_test)  # uses train statistics only
 ```
 
 Save the fitted cleaner with the model artifact:
@@ -197,11 +197,9 @@ contract = DataContract(
     ],
 )
 
-cleaner = DataCleaner(
-    CleanerConfig(contract=contract, on_drift="warn")
-).fit(X_train)
+cleaner = DataCleaner(CleanerConfig(contract=contract, on_drift="warn")).fit(X_train)
 
-X_test_clean = cleaner.transform(X_test)   # drift is detected, never mutates
+X_test_clean = cleaner.transform(X_test)  # drift is detected, never mutates
 print(cleaner.drift_report())
 ```
 
@@ -236,13 +234,68 @@ Use `role="ignore"` to drop operational columns from the output.
 Contracts can be version-controlled and shared between training and serving:
 
 ```python
-contract.to_json()                       # store it with the model artifact
+contract.to_json()  # store it with the model artifact
 restored = DataContract.from_json(payload)
 
 # Optional YAML support requires `pip install pyyaml`
 # restored = DataContract.from_yaml(contract.to_yaml())
 ```
 
+## Audit trail, manifests, and fingerprints
+
+Every learned or configured decision is recorded during `fit()` and exposed as
+a tidy DataFrame:
+
+```python
+log = cleaner.decision_log()
+# columns: column, action, reason, source, before, after
+log[log["action"] == "cap_outliers"]
+```
+
+`source` says whether a rule was `learned` from the training data, supplied via
+`config`, declared in a `contract`, or `inferred` by the library. Imputation
+values, outlier bounds, datetime expansion, missing indicators, dropped
+constants, and identifier handling are all explained instead of hidden.
+
+Build a JSON-serializable manifest for logs or a model registry:
+
+```python
+import json
+
+manifest = cleaner.to_manifest(X_train, include_fingerprint=True)
+# library_version, artifact_version, environment, config, contract,
+# input_schema, output_columns, fitted, decisions, fingerprint
+
+json.dumps(manifest)  # safe to ship to a registry or a log line
+```
+
+`save()` embeds the manifest and the training fingerprint in the artifact, and
+`load()` restores them:
+
+```python
+cleaner.save("cleaner.joblib")
+restored = DataCleaner.load("cleaner.joblib")
+restored.artifact_manifest()
+```
+
+Fingerprints are stable SHA-256 hashes of the schema plus summary statistics.
+They never hash raw values, so they are safe to log:
+
+```python
+from data_cleaner import (
+    assert_schema_stable,
+    compare_fingerprints,
+    fingerprint_dataframe,
+)
+
+expected = fingerprint_dataframe(X_train)
+same = compare_fingerprints(expected, fingerprint_dataframe(X_new))
+assert_schema_stable(X_new, expected)  # raises ValueError on schema/stat drift
+```
+
+Set `CleanerConfig(fingerprint_on_fit=True)` to compute the training fingerprint
+once during `fit()`, so `to_manifest(include_fingerprint=True)` needs no data.
+
 ## scikit-learn Pipeline
 
 ```python
@@ -281,11 +334,19 @@ predictions = pipeline.predict(X_test)
 | --- | --- |
 | `CleanerConfig` | Validated cleaning configuration |
 | `DataCleaner` | Fit/transform cleaner with save/load and drift reporting |
+| `CleaningDecision` | One auditable learned or configured decision |
 | `ColumnSpec` | Declarative expectations and semantics for one column |
 | `DataContract` | A set of column specs, with inference and (de)serialization |
 | `DriftReport` | Findings from a contract check, with a Markdown summary |
 | `DataCleaner.drift_report()` | Drift findings from the last `transform` |
 | `DataCleaner.fitted_contract()` | The contract used by the fitted cleaner |
+| `DataCleaner.decision_log()` | Tidy DataFrame of cleaning decisions |
+| `DataCleaner.to_manifest()` | JSON-serializable artifact manifest |
+| `DataCleaner.artifact_manifest()` | Manifest stored in a loaded artifact |
+| `DataCleaner.fingerprint()` | Stable schema + statistics fingerprint |
+| `fingerprint_dataframe` | Fingerprint any DataFrame for lineage |
+| `compare_fingerprints` | Compare two fingerprints |
+| `assert_schema_stable` | Raise when a frame's fingerprint changed |
 | `profile_dataframe` | Column-level missing/type/cardinality profile |
 | `dataset_summary` | Row/column/missing/duplicate summary |
 | `drop_duplicate_rows` | Explicit duplicate removal helper |
@@ -307,16 +368,29 @@ predictions = pipeline.predict(X_test)
   represented by the generated feature NaNs plus an optional missing indicator.
 - IQR capping is skipped automatically when the IQR is zero, so constant or
   near-constant numeric columns are not accidentally destroyed.
+- Persist `cleaner.to_manifest(X_train, include_fingerprint=True)` alongside the
+  model so you can audit exactly which cleaning rules produced an artifact.
+- `decision_log()` explains every learned or configured rule; ship it to your
+  experiment tracker when a model's preprocessing changes.
 
 ## Project layout
 
 ```text
 .
-├── data_cleaner.py                    # reusable library module
+├── src/data_cleaner/                  # typed, pip-installable package
+│   ├── __init__.py                    # public API re-exports
+│   ├── _version.py                    # version metadata
+│   ├── _roles.py                      # semantic roles and dtype helpers
+│   ├── contract.py                    # ColumnSpec / DataContract / DriftReport
+│   ├── config.py                      # CleanerConfig
+│   ├── cleaner.py                     # DataCleaner
+│   ├── profiling.py                   # profiling and validation helpers
+│   └── py.typed                       # PEP 561 marker
 ├── Universal_Data_Cleaner.ipynb       # end-to-end reference workflow
 ├── tests/test_data_cleaner.py         # unit and integration tests
 ├── pyproject.toml                     # package metadata and dev tooling
 ├── uv.lock                            # reproducible dependency lock
+├── .pre-commit-config.yaml            # local lint/format/type hooks
 ├── README.md
 ├── ROADMAP.md
 ├── CHANGELOG.md
@@ -325,6 +399,10 @@ predictions = pipeline.predict(X_test)
 └── LICENSE
 ```
 
+The package ships a `py.typed` marker, so `mypy`, `pyright`, and IDE tooling
+pick up the type annotations automatically. Linting (`ruff`), formatting, and
+type checking (`mypy`) run in CI and via `pre-commit`.
+
 ## Contributing
 
 Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for the local

{universal_data_cleaner-2.1.0 → universal_data_cleaner-2.3.0}/README.md

@@ -120,9 +120,9 @@ For a column that should be numeric even though it arrived as dirty strings:
 
 ```python
 config = CleanerConfig(
-    numeric_columns=("salary",),          # force numeric parsing
-    datetime_columns=("joined",),        # parse dates instead of treating them as categories
-    unseen_category="keep",              # or "fill" to replace unseen labels
+    numeric_columns=("salary",),  # force numeric parsing
+    datetime_columns=("joined",),  # parse dates instead of treating them as categories
+    unseen_category="keep",  # or "fill" to replace unseen labels
 )
 ```
 
@@ -138,7 +138,7 @@ X_train, X_test, y_train, y_test = train_test_split(
 cleaner = DataCleaner(config).fit(X_train)
 
 X_train_clean = cleaner.transform(X_train)
-X_test_clean = cleaner.transform(X_test)   # uses train statistics only
+X_test_clean = cleaner.transform(X_test)  # uses train statistics only
 ```
 
 Save the fitted cleaner with the model artifact:
@@ -164,11 +164,9 @@ contract = DataContract(
     ],
 )
 
-cleaner = DataCleaner(
-    CleanerConfig(contract=contract, on_drift="warn")
-).fit(X_train)
+cleaner = DataCleaner(CleanerConfig(contract=contract, on_drift="warn")).fit(X_train)
 
-X_test_clean = cleaner.transform(X_test)   # drift is detected, never mutates
+X_test_clean = cleaner.transform(X_test)  # drift is detected, never mutates
 print(cleaner.drift_report())
 ```
 
@@ -203,13 +201,68 @@ Use `role="ignore"` to drop operational columns from the output.
 Contracts can be version-controlled and shared between training and serving:
 
 ```python
-contract.to_json()                       # store it with the model artifact
+contract.to_json()  # store it with the model artifact
 restored = DataContract.from_json(payload)
 
 # Optional YAML support requires `pip install pyyaml`
 # restored = DataContract.from_yaml(contract.to_yaml())
 ```
 
+## Audit trail, manifests, and fingerprints
+
+Every learned or configured decision is recorded during `fit()` and exposed as
+a tidy DataFrame:
+
+```python
+log = cleaner.decision_log()
+# columns: column, action, reason, source, before, after
+log[log["action"] == "cap_outliers"]
+```
+
+`source` says whether a rule was `learned` from the training data, supplied via
+`config`, declared in a `contract`, or `inferred` by the library. Imputation
+values, outlier bounds, datetime expansion, missing indicators, dropped
+constants, and identifier handling are all explained instead of hidden.
+
+Build a JSON-serializable manifest for logs or a model registry:
+
+```python
+import json
+
+manifest = cleaner.to_manifest(X_train, include_fingerprint=True)
+# library_version, artifact_version, environment, config, contract,
+# input_schema, output_columns, fitted, decisions, fingerprint
+
+json.dumps(manifest)  # safe to ship to a registry or a log line
+```
+
+`save()` embeds the manifest and the training fingerprint in the artifact, and
+`load()` restores them:
+
+```python
+cleaner.save("cleaner.joblib")
+restored = DataCleaner.load("cleaner.joblib")
+restored.artifact_manifest()
+```
+
+Fingerprints are stable SHA-256 hashes of the schema plus summary statistics.
+They never hash raw values, so they are safe to log:
+
+```python
+from data_cleaner import (
+    assert_schema_stable,
+    compare_fingerprints,
+    fingerprint_dataframe,
+)
+
+expected = fingerprint_dataframe(X_train)
+same = compare_fingerprints(expected, fingerprint_dataframe(X_new))
+assert_schema_stable(X_new, expected)  # raises ValueError on schema/stat drift
+```
+
+Set `CleanerConfig(fingerprint_on_fit=True)` to compute the training fingerprint
+once during `fit()`, so `to_manifest(include_fingerprint=True)` needs no data.
+
 ## scikit-learn Pipeline
 
 ```python
@@ -248,11 +301,19 @@ predictions = pipeline.predict(X_test)
 | --- | --- |
 | `CleanerConfig` | Validated cleaning configuration |
 | `DataCleaner` | Fit/transform cleaner with save/load and drift reporting |
+| `CleaningDecision` | One auditable learned or configured decision |
 | `ColumnSpec` | Declarative expectations and semantics for one column |
 | `DataContract` | A set of column specs, with inference and (de)serialization |
 | `DriftReport` | Findings from a contract check, with a Markdown summary |
 | `DataCleaner.drift_report()` | Drift findings from the last `transform` |
 | `DataCleaner.fitted_contract()` | The contract used by the fitted cleaner |
+| `DataCleaner.decision_log()` | Tidy DataFrame of cleaning decisions |
+| `DataCleaner.to_manifest()` | JSON-serializable artifact manifest |
+| `DataCleaner.artifact_manifest()` | Manifest stored in a loaded artifact |
+| `DataCleaner.fingerprint()` | Stable schema + statistics fingerprint |
+| `fingerprint_dataframe` | Fingerprint any DataFrame for lineage |
+| `compare_fingerprints` | Compare two fingerprints |
+| `assert_schema_stable` | Raise when a frame's fingerprint changed |
 | `profile_dataframe` | Column-level missing/type/cardinality profile |
 | `dataset_summary` | Row/column/missing/duplicate summary |
 | `drop_duplicate_rows` | Explicit duplicate removal helper |
@@ -274,16 +335,29 @@ predictions = pipeline.predict(X_test)
   represented by the generated feature NaNs plus an optional missing indicator.
 - IQR capping is skipped automatically when the IQR is zero, so constant or
   near-constant numeric columns are not accidentally destroyed.
+- Persist `cleaner.to_manifest(X_train, include_fingerprint=True)` alongside the
+  model so you can audit exactly which cleaning rules produced an artifact.
+- `decision_log()` explains every learned or configured rule; ship it to your
+  experiment tracker when a model's preprocessing changes.
 
 ## Project layout
 
 ```text
 .
-├── data_cleaner.py                    # reusable library module
+├── src/data_cleaner/                  # typed, pip-installable package
+│   ├── __init__.py                    # public API re-exports
+│   ├── _version.py                    # version metadata
+│   ├── _roles.py                      # semantic roles and dtype helpers
+│   ├── contract.py                    # ColumnSpec / DataContract / DriftReport
+│   ├── config.py                      # CleanerConfig
+│   ├── cleaner.py                     # DataCleaner
+│   ├── profiling.py                   # profiling and validation helpers
+│   └── py.typed                       # PEP 561 marker
 ├── Universal_Data_Cleaner.ipynb       # end-to-end reference workflow
 ├── tests/test_data_cleaner.py         # unit and integration tests
 ├── pyproject.toml                     # package metadata and dev tooling
 ├── uv.lock                            # reproducible dependency lock
+├── .pre-commit-config.yaml            # local lint/format/type hooks
 ├── README.md
 ├── ROADMAP.md
 ├── CHANGELOG.md
@@ -292,6 +366,10 @@ predictions = pipeline.predict(X_test)
 └── LICENSE
 ```
 
+The package ships a `py.typed` marker, so `mypy`, `pyright`, and IDE tooling
+pick up the type annotations automatically. Linting (`ruff`), formatting, and
+type checking (`mypy`) run in CI and via `pre-commit`.
+
 ## Contributing
 
 Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for the local

{universal_data_cleaner-2.1.0 → universal_data_cleaner-2.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "universal-data-cleaner"
-version = "2.1.0"
+version = "2.3.0"
 description = "Production-ready, scikit-learn-compatible data cleaning transformer for pandas."
 readme = "README.md"
 license = "MIT"
@@ -53,14 +53,54 @@ Changelog = "https://github.com/devlprkhan/universal-data-cleaner/blob/main/CHAN
 dev = [
     "ipykernel>=7.3.0",
     "jupyterlab>=4.6.4",
+    "mypy>=2.4.0",
     "nbclient>=0.11.0",
     "nbformat>=5.11.1",
+    "pandas-stubs>=2.3.3.260113",
+    "pre-commit>=4.6.2",
     "pytest>=9.1.1",
+    "ruff>=0.17.1",
 ]
 
-[tool.setuptools]
-py-modules = ["data_cleaner"]
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+data_cleaner = ["py.typed"]
 
 [tool.pytest.ini_options]
-pythonpath = ["."]
+pythonpath = ["src"]
 testpaths = ["tests"]
+addopts = "-q"
+
+[tool.ruff]
+line-length = 88
+target-version = "py310"
+src = ["src", "tests"]
+extend-exclude = ["*.ipynb"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP", "C4", "SIM", "RUF"]
+ignore = ["E501"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["B011"]
+
+[tool.ruff.format]
+quote-style = "double"
+
+[tool.mypy]
+python_version = "3.10"
+check_untyped_defs = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_unreachable = true
+show_error_codes = true
+ignore_missing_imports = true
+mypy_path = "src"
+files = ["src", "tests"]
+
+[[tool.mypy.overrides]]
+module = ["sklearn.*", "yaml", "joblib.*"]
+ignore_missing_imports = true

universal_data_cleaner-2.3.0/src/data_cleaner/__init__.py

@@ -0,0 +1,47 @@
+"""Universal Data Cleaner — production-oriented tabular cleaning toolkit.
+
+The package is intentionally dependency-light:
+
+- pandas + numpy for data handling
+- scikit-learn base classes for Pipeline / clone compatibility
+- joblib for artifact persistence
+
+The public API is re-exported here so ``import data_cleaner`` keeps working
+exactly as it did when the library shipped as a single module.
+"""
+
+from __future__ import annotations
+
+from ._version import __version__
+from .artifacts import (
+    assert_schema_stable,
+    compare_fingerprints,
+    fingerprint_dataframe,
+)
+from .cleaner import DataCleaner
+from .config import CleanerConfig
+from .contract import ColumnSpec, DataContract, DriftReport
+from .decisions import CleaningDecision
+from .profiling import (
+    assert_frame_clean,
+    dataset_summary,
+    drop_duplicate_rows,
+    profile_dataframe,
+)
+
+__all__ = [
+    "CleanerConfig",
+    "CleaningDecision",
+    "ColumnSpec",
+    "DataCleaner",
+    "DataContract",
+    "DriftReport",
+    "__version__",
+    "assert_frame_clean",
+    "assert_schema_stable",
+    "compare_fingerprints",
+    "dataset_summary",
+    "drop_duplicate_rows",
+    "fingerprint_dataframe",
+    "profile_dataframe",
+]

universal_data_cleaner-2.3.0/src/data_cleaner/_roles.py

@@ -0,0 +1,134 @@
+"""Semantic roles, dtype normalization, and identifier helpers."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from pandas.api import types as ptypes
+
+_ALLOWED_DATETIME_FEATURES = {
+    "year",
+    "month",
+    "day",
+    "dayofweek",
+    "quarter",
+    "dayofyear",
+    "weekofyear",
+    "is_weekend",
+    "hour",
+    "minute",
+}
+
+_ROLE_ALIASES = {
+    "numeric": "numeric",
+    "measure": "numeric",
+    "number": "numeric",
+    "float": "numeric",
+    "int": "numeric",
+    "integer": "numeric",
+    "category": "category",
+    "categorical": "category",
+    "object": "category",
+    "string": "category",
+    "str": "category",
+    "boolean": "boolean",
+    "bool": "boolean",
+    "datetime": "datetime",
+    "date": "datetime",
+    "timestamp": "datetime",
+    "id": "id",
+    "identifier": "id",
+    "text": "text",
+    "ignore": "ignore",
+    "drop": "ignore",
+}
+
+_ROLE_TO_KIND = {
+    "numeric": "numeric",
+    "category": "categorical",
+    "boolean": "boolean",
+    "datetime": "datetime",
+    "id": "categorical",
+    "text": "categorical",
+    "ignore": None,
+}
+
+_DRIFT_COLUMNS = ["column", "check", "severity", "expected", "actual", "message"]
+
+_ID_STRING_KINDS = (int, np.integer)
+
+
+def _normalize_role(value: object) -> str | None:
+    """Map role aliases to a canonical role name."""
+    if value is None:
+        return None
+    key = str(value).strip().lower()
+    if key not in _ROLE_ALIASES:
+        valid = sorted(set(_ROLE_ALIASES.values()))
+        raise ValueError(f"Unsupported column role '{value}'. Valid roles: {valid}")
+    return _ROLE_ALIASES[key]
+
+
+def _normalize_kind(value: object) -> str | None:
+    """Map dtype strings to a broad kind: numeric/categorical/boolean/datetime."""
+    if value is None:
+        return None
+    key = str(value).strip().lower()
+    mapping = {
+        "numeric": "numeric",
+        "number": "numeric",
+        "int": "numeric",
+        "integer": "numeric",
+        "float": "numeric",
+        "double": "numeric",
+        "int64": "numeric",
+        "int32": "numeric",
+        "int16": "numeric",
+        "int8": "numeric",
+        "float64": "numeric",
+        "float32": "numeric",
+        "boolean": "boolean",
+        "bool": "boolean",
+        "datetime": "datetime",
+        "datetime64": "datetime",
+        "datetime64[ns]": "datetime",
+        "date": "datetime",
+        "timestamp": "datetime",
+        "categorical": "categorical",
+        "category": "categorical",
+        "object": "categorical",
+        "string": "categorical",
+        "str": "categorical",
+        "text": "categorical",
+        "id": "categorical",
+    }
+    return mapping.get(key, key)
+
+
+def _series_kind(series: pd.Series) -> str:
+    """Return the broad semantic kind of a pandas Series."""
+    if ptypes.is_bool_dtype(series):
+        return "boolean"
+    if ptypes.is_datetime64_any_dtype(series):
+        return "datetime"
+    if ptypes.is_numeric_dtype(series):
+        return "numeric"
+    return "categorical"
+
+
+def _stringify_identifier(value: Any) -> object:
+    """Render an identifier without a trailing ``.0`` while preserving NaN."""
+    if value is None:
+        return value
+    try:
+        if pd.isna(value):
+            return value
+    except (TypeError, ValueError):
+        pass
+    if isinstance(value, _ID_STRING_KINDS):
+        return str(int(value))
+    if isinstance(value, (float, np.floating)) and float(value).is_integer():
+        return str(int(value))
+    return str(value)

universal_data_cleaner-2.3.0/src/data_cleaner/_version.py

@@ -0,0 +1,4 @@
+"""Single source of truth for the package version."""
+
+__version__ = "2.3.0"
+_ARTIFACT_VERSION = 2