universal-data-cleaner 2.2.0__tar.gz → 2.4.0__tar.gz

{universal_data_cleaner-2.2.0/src/universal_data_cleaner.egg-info → universal_data_cleaner-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: universal-data-cleaner
-Version: 2.2.0
+Version: 2.4.0
 Summary: Production-ready, scikit-learn-compatible data cleaning transformer for pandas.
 Author: Haseeb Khan
 License-Expression: MIT
@@ -241,6 +241,165 @@ restored = DataContract.from_json(payload)
 # restored = DataContract.from_yaml(contract.to_yaml())
 ```
 
+## Expectations, joins, and referential integrity
+
+### Expectations and validation reports
+
+Run lightweight expectations against any batch and get a shareable report:
+
+```python
+from data_cleaner import (
+    between,
+    has_dtype,
+    in_set,
+    matches_pattern,
+    not_null,
+    row_count_between,
+    unique,
+    validate,
+)
+
+report = validate(
+    orders_new,
+    [
+        not_null("order_id"),
+        unique("order_id"),
+        in_set("status", ["new", "paid", "shipped"]),
+        between("amount", 0, 100_000),
+        matches_pattern("email", r"[^@]+@[^@]+\.[^@]+"),
+        has_dtype("created_at", "datetime"),
+        row_count_between(min_rows=1),
+    ],
+)
+print(report.to_markdown())
+report.raise_if_failed()
+```
+
+`report.success` is true when no error-severity expectation failed.
+`report.failures` includes warnings too, so use `severity="warning"` for soft
+checks. `has_dtype` accepts broad kinds (`numeric`, `categorical`, `boolean`,
+`datetime`) or an exact pandas dtype such as `float64`.
+
+Contracts can validate a batch directly and return the same report type:
+
+```python
+from data_cleaner import DataContract
+
+contract = DataContract.from_frame(orders_train)
+result = contract.validate(orders_new)
+result.failures.to_markdown()
+```
+
+`Contract` is a shorthand alias for `DataContract`.
+
+### Join diagnostics
+
+```python
+from data_cleaner import diagnose_join
+
+report = diagnose_join(orders, customers, on="customer_id", how="left")
+
+report.safe  # False on cardinality mismatch, left/right row multiplication, or null keys
+report.cardinality  # e.g. "1:m"
+report.row_multiplier  # output rows / left rows
+report.unmatched_left
+report.unmatched_right
+report.warnings
+report.raise_if_unsafe()
+```
+
+`diagnose_join` defaults to `how="left"` and `validate="1:1"`. It reports
+duplicate keys, null keys, unmatched rows, and the actual output row count
+without materializing the full joined frame.
+
+### Referential integrity
+
+```python
+from data_cleaner import check_referential_integrity
+
+report = check_referential_integrity(
+    orders, customers, child_key="customer_id", parent_key="id"
+)
+report.orphan_rows
+report.orphan_keys
+report.raise_if_invalid()
+```
+
+Null child keys are reported but allowed by default (`allow_null_keys=True`).
+Duplicate parent keys make the relationship invalid because the parent key is
+not a unique reference target.
+
+### Business-key deduplication
+
+```python
+from data_cleaner import deduplicate_by_business_key
+
+deduped, report = deduplicate_by_business_key(
+    orders, key="order_id", keep="last", tie_breaker="updated_at"
+)
+report.removed_rows
+report.removed  # the actual rows that were dropped
+```
+
+The returned frame preserves the input row order and index. `tie_breaker`
+prefers non-null values, and null business keys are reported explicitly.
+The input frame is never mutated.
+
+## Audit trail, manifests, and fingerprints
+
+Every learned or configured decision is recorded during `fit()` and exposed as
+a tidy DataFrame:
+
+```python
+log = cleaner.decision_log()
+# columns: column, action, reason, source, before, after
+log[log["action"] == "cap_outliers"]
+```
+
+`source` says whether a rule was `learned` from the training data, supplied via
+`config`, declared in a `contract`, or `inferred` by the library. Imputation
+values, outlier bounds, datetime expansion, missing indicators, dropped
+constants, and identifier handling are all explained instead of hidden.
+
+Build a JSON-serializable manifest for logs or a model registry:
+
+```python
+import json
+
+manifest = cleaner.to_manifest(X_train, include_fingerprint=True)
+# library_version, artifact_version, environment, config, contract,
+# input_schema, output_columns, fitted, decisions, fingerprint
+
+json.dumps(manifest)  # safe to ship to a registry or a log line
+```
+
+`save()` embeds the manifest and the training fingerprint in the artifact, and
+`load()` restores them:
+
+```python
+cleaner.save("cleaner.joblib")
+restored = DataCleaner.load("cleaner.joblib")
+restored.artifact_manifest()
+```
+
+Fingerprints are stable SHA-256 hashes of the schema plus summary statistics.
+They never hash raw values, so they are safe to log:
+
+```python
+from data_cleaner import (
+    assert_schema_stable,
+    compare_fingerprints,
+    fingerprint_dataframe,
+)
+
+expected = fingerprint_dataframe(X_train)
+same = compare_fingerprints(expected, fingerprint_dataframe(X_new))
+assert_schema_stable(X_new, expected)  # raises ValueError on schema/stat drift
+```
+
+Set `CleanerConfig(fingerprint_on_fit=True)` to compute the training fingerprint
+once during `fit()`, so `to_manifest(include_fingerprint=True)` needs no data.
+
 ## scikit-learn Pipeline
 
 ```python
@@ -279,11 +438,33 @@ predictions = pipeline.predict(X_test)
 | --- | --- |
 | `CleanerConfig` | Validated cleaning configuration |
 | `DataCleaner` | Fit/transform cleaner with save/load and drift reporting |
+| `CleaningDecision` | One auditable learned or configured decision |
 | `ColumnSpec` | Declarative expectations and semantics for one column |
 | `DataContract` | A set of column specs, with inference and (de)serialization |
+| `Contract` | Shorthand alias for `DataContract` |
+| `DataContract.validate()` | Check a batch and return a `ValidationReport` |
 | `DriftReport` | Findings from a contract check, with a Markdown summary |
+| `Expectation` | One reusable expectation produced by a factory |
+| `validate` | Evaluate expectations against a frame |
+| `not_null`, `unique`, `in_set`, `between` | Expectation factories |
+| `matches_pattern`, `has_dtype` | Expectation factories |
+| `row_count_between` | Frame-level row-count expectation |
+| `ValidationReport` | Pass/fail results, failures table, Markdown, and raise helper |
+| `diagnose_join` | Diagnose key duplication, cardinality, and row multiplication |
+| `JoinDiagnosis` | Join report with multiplier, unmatched counts, and warnings |
+| `check_referential_integrity` | Find orphan child keys and non-unique parent keys |
+| `ReferentialIntegrityReport` | Orphan counts/keys, null keys, and validity |
+| `deduplicate_by_business_key` | Deduplicate by key and report removed rows |
+| `DeduplicationReport` | Removed rows, duplicate-key counts, and tie-breaker details |
 | `DataCleaner.drift_report()` | Drift findings from the last `transform` |
 | `DataCleaner.fitted_contract()` | The contract used by the fitted cleaner |
+| `DataCleaner.decision_log()` | Tidy DataFrame of cleaning decisions |
+| `DataCleaner.to_manifest()` | JSON-serializable artifact manifest |
+| `DataCleaner.artifact_manifest()` | Manifest stored in a loaded artifact |
+| `DataCleaner.fingerprint()` | Stable schema + statistics fingerprint |
+| `fingerprint_dataframe` | Fingerprint any DataFrame for lineage |
+| `compare_fingerprints` | Compare two fingerprints |
+| `assert_schema_stable` | Raise when a frame's fingerprint changed |
 | `profile_dataframe` | Column-level missing/type/cardinality profile |
 | `dataset_summary` | Row/column/missing/duplicate summary |
 | `drop_duplicate_rows` | Explicit duplicate removal helper |
@@ -305,6 +486,10 @@ predictions = pipeline.predict(X_test)
   represented by the generated feature NaNs plus an optional missing indicator.
 - IQR capping is skipped automatically when the IQR is zero, so constant or
   near-constant numeric columns are not accidentally destroyed.
+- Persist `cleaner.to_manifest(X_train, include_fingerprint=True)` alongside the
+  model so you can audit exactly which cleaning rules produced an artifact.
+- `decision_log()` explains every learned or configured rule; ship it to your
+  experiment tracker when a model's preprocessing changes.
 
 ## Project layout
 
@@ -314,13 +499,18 @@ predictions = pipeline.predict(X_test)
 │   ├── __init__.py                    # public API re-exports
 │   ├── _version.py                    # version metadata
 │   ├── _roles.py                      # semantic roles and dtype helpers
+│   ├── artifacts.py                   # fingerprints and runtime metadata
 │   ├── contract.py                    # ColumnSpec / DataContract / DriftReport
 │   ├── config.py                      # CleanerConfig
 │   ├── cleaner.py                     # DataCleaner
+│   ├── decisions.py                   # CleaningDecision and decision log
+│   ├── integration.py                 # joins, referential integrity, dedup
 │   ├── profiling.py                   # profiling and validation helpers
+│   ├── validation.py                  # expectations and ValidationReport
 │   └── py.typed                       # PEP 561 marker
 ├── Universal_Data_Cleaner.ipynb       # end-to-end reference workflow
-├── tests/test_data_cleaner.py         # unit and integration tests
+├── tests/test_data_cleaner.py         # core unit and integration tests
+├── tests/test_validation.py           # expectations and integration tests
 ├── pyproject.toml                     # package metadata and dev tooling
 ├── uv.lock                            # reproducible dependency lock
 ├── .pre-commit-config.yaml            # local lint/format/type hooks

{universal_data_cleaner-2.2.0 → universal_data_cleaner-2.4.0}/README.md

@@ -208,6 +208,165 @@ restored = DataContract.from_json(payload)
 # restored = DataContract.from_yaml(contract.to_yaml())
 ```
 
+## Expectations, joins, and referential integrity
+
+### Expectations and validation reports
+
+Run lightweight expectations against any batch and get a shareable report:
+
+```python
+from data_cleaner import (
+    between,
+    has_dtype,
+    in_set,
+    matches_pattern,
+    not_null,
+    row_count_between,
+    unique,
+    validate,
+)
+
+report = validate(
+    orders_new,
+    [
+        not_null("order_id"),
+        unique("order_id"),
+        in_set("status", ["new", "paid", "shipped"]),
+        between("amount", 0, 100_000),
+        matches_pattern("email", r"[^@]+@[^@]+\.[^@]+"),
+        has_dtype("created_at", "datetime"),
+        row_count_between(min_rows=1),
+    ],
+)
+print(report.to_markdown())
+report.raise_if_failed()
+```
+
+`report.success` is true when no error-severity expectation failed.
+`report.failures` includes warnings too, so use `severity="warning"` for soft
+checks. `has_dtype` accepts broad kinds (`numeric`, `categorical`, `boolean`,
+`datetime`) or an exact pandas dtype such as `float64`.
+
+Contracts can validate a batch directly and return the same report type:
+
+```python
+from data_cleaner import DataContract
+
+contract = DataContract.from_frame(orders_train)
+result = contract.validate(orders_new)
+result.failures.to_markdown()
+```
+
+`Contract` is a shorthand alias for `DataContract`.
+
+### Join diagnostics
+
+```python
+from data_cleaner import diagnose_join
+
+report = diagnose_join(orders, customers, on="customer_id", how="left")
+
+report.safe  # False on cardinality mismatch, left/right row multiplication, or null keys
+report.cardinality  # e.g. "1:m"
+report.row_multiplier  # output rows / left rows
+report.unmatched_left
+report.unmatched_right
+report.warnings
+report.raise_if_unsafe()
+```
+
+`diagnose_join` defaults to `how="left"` and `validate="1:1"`. It reports
+duplicate keys, null keys, unmatched rows, and the actual output row count
+without materializing the full joined frame.
+
+### Referential integrity
+
+```python
+from data_cleaner import check_referential_integrity
+
+report = check_referential_integrity(
+    orders, customers, child_key="customer_id", parent_key="id"
+)
+report.orphan_rows
+report.orphan_keys
+report.raise_if_invalid()
+```
+
+Null child keys are reported but allowed by default (`allow_null_keys=True`).
+Duplicate parent keys make the relationship invalid because the parent key is
+not a unique reference target.
+
+### Business-key deduplication
+
+```python
+from data_cleaner import deduplicate_by_business_key
+
+deduped, report = deduplicate_by_business_key(
+    orders, key="order_id", keep="last", tie_breaker="updated_at"
+)
+report.removed_rows
+report.removed  # the actual rows that were dropped
+```
+
+The returned frame preserves the input row order and index. `tie_breaker`
+prefers non-null values, and null business keys are reported explicitly.
+The input frame is never mutated.
+
+## Audit trail, manifests, and fingerprints
+
+Every learned or configured decision is recorded during `fit()` and exposed as
+a tidy DataFrame:
+
+```python
+log = cleaner.decision_log()
+# columns: column, action, reason, source, before, after
+log[log["action"] == "cap_outliers"]
+```
+
+`source` says whether a rule was `learned` from the training data, supplied via
+`config`, declared in a `contract`, or `inferred` by the library. Imputation
+values, outlier bounds, datetime expansion, missing indicators, dropped
+constants, and identifier handling are all explained instead of hidden.
+
+Build a JSON-serializable manifest for logs or a model registry:
+
+```python
+import json
+
+manifest = cleaner.to_manifest(X_train, include_fingerprint=True)
+# library_version, artifact_version, environment, config, contract,
+# input_schema, output_columns, fitted, decisions, fingerprint
+
+json.dumps(manifest)  # safe to ship to a registry or a log line
+```
+
+`save()` embeds the manifest and the training fingerprint in the artifact, and
+`load()` restores them:
+
+```python
+cleaner.save("cleaner.joblib")
+restored = DataCleaner.load("cleaner.joblib")
+restored.artifact_manifest()
+```
+
+Fingerprints are stable SHA-256 hashes of the schema plus summary statistics.
+They never hash raw values, so they are safe to log:
+
+```python
+from data_cleaner import (
+    assert_schema_stable,
+    compare_fingerprints,
+    fingerprint_dataframe,
+)
+
+expected = fingerprint_dataframe(X_train)
+same = compare_fingerprints(expected, fingerprint_dataframe(X_new))
+assert_schema_stable(X_new, expected)  # raises ValueError on schema/stat drift
+```
+
+Set `CleanerConfig(fingerprint_on_fit=True)` to compute the training fingerprint
+once during `fit()`, so `to_manifest(include_fingerprint=True)` needs no data.
+
 ## scikit-learn Pipeline
 
 ```python
@@ -246,11 +405,33 @@ predictions = pipeline.predict(X_test)
 | --- | --- |
 | `CleanerConfig` | Validated cleaning configuration |
 | `DataCleaner` | Fit/transform cleaner with save/load and drift reporting |
+| `CleaningDecision` | One auditable learned or configured decision |
 | `ColumnSpec` | Declarative expectations and semantics for one column |
 | `DataContract` | A set of column specs, with inference and (de)serialization |
+| `Contract` | Shorthand alias for `DataContract` |
+| `DataContract.validate()` | Check a batch and return a `ValidationReport` |
 | `DriftReport` | Findings from a contract check, with a Markdown summary |
+| `Expectation` | One reusable expectation produced by a factory |
+| `validate` | Evaluate expectations against a frame |
+| `not_null`, `unique`, `in_set`, `between` | Expectation factories |
+| `matches_pattern`, `has_dtype` | Expectation factories |
+| `row_count_between` | Frame-level row-count expectation |
+| `ValidationReport` | Pass/fail results, failures table, Markdown, and raise helper |
+| `diagnose_join` | Diagnose key duplication, cardinality, and row multiplication |
+| `JoinDiagnosis` | Join report with multiplier, unmatched counts, and warnings |
+| `check_referential_integrity` | Find orphan child keys and non-unique parent keys |
+| `ReferentialIntegrityReport` | Orphan counts/keys, null keys, and validity |
+| `deduplicate_by_business_key` | Deduplicate by key and report removed rows |
+| `DeduplicationReport` | Removed rows, duplicate-key counts, and tie-breaker details |
 | `DataCleaner.drift_report()` | Drift findings from the last `transform` |
 | `DataCleaner.fitted_contract()` | The contract used by the fitted cleaner |
+| `DataCleaner.decision_log()` | Tidy DataFrame of cleaning decisions |
+| `DataCleaner.to_manifest()` | JSON-serializable artifact manifest |
+| `DataCleaner.artifact_manifest()` | Manifest stored in a loaded artifact |
+| `DataCleaner.fingerprint()` | Stable schema + statistics fingerprint |
+| `fingerprint_dataframe` | Fingerprint any DataFrame for lineage |
+| `compare_fingerprints` | Compare two fingerprints |
+| `assert_schema_stable` | Raise when a frame's fingerprint changed |
 | `profile_dataframe` | Column-level missing/type/cardinality profile |
 | `dataset_summary` | Row/column/missing/duplicate summary |
 | `drop_duplicate_rows` | Explicit duplicate removal helper |
@@ -272,6 +453,10 @@ predictions = pipeline.predict(X_test)
   represented by the generated feature NaNs plus an optional missing indicator.
 - IQR capping is skipped automatically when the IQR is zero, so constant or
   near-constant numeric columns are not accidentally destroyed.
+- Persist `cleaner.to_manifest(X_train, include_fingerprint=True)` alongside the
+  model so you can audit exactly which cleaning rules produced an artifact.
+- `decision_log()` explains every learned or configured rule; ship it to your
+  experiment tracker when a model's preprocessing changes.
 
 ## Project layout
 
@@ -281,13 +466,18 @@ predictions = pipeline.predict(X_test)
 │   ├── __init__.py                    # public API re-exports
 │   ├── _version.py                    # version metadata
 │   ├── _roles.py                      # semantic roles and dtype helpers
+│   ├── artifacts.py                   # fingerprints and runtime metadata
 │   ├── contract.py                    # ColumnSpec / DataContract / DriftReport
 │   ├── config.py                      # CleanerConfig
 │   ├── cleaner.py                     # DataCleaner
+│   ├── decisions.py                   # CleaningDecision and decision log
+│   ├── integration.py                 # joins, referential integrity, dedup
 │   ├── profiling.py                   # profiling and validation helpers
+│   ├── validation.py                  # expectations and ValidationReport
 │   └── py.typed                       # PEP 561 marker
 ├── Universal_Data_Cleaner.ipynb       # end-to-end reference workflow
-├── tests/test_data_cleaner.py         # unit and integration tests
+├── tests/test_data_cleaner.py         # core unit and integration tests
+├── tests/test_validation.py           # expectations and integration tests
 ├── pyproject.toml                     # package metadata and dev tooling
 ├── uv.lock                            # reproducible dependency lock
 ├── .pre-commit-config.yaml            # local lint/format/type hooks

{universal_data_cleaner-2.2.0 → universal_data_cleaner-2.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "universal-data-cleaner"
-version = "2.2.0"
+version = "2.4.0"
 description = "Production-ready, scikit-learn-compatible data cleaning transformer for pandas."
 readme = "README.md"
 license = "MIT"

universal_data_cleaner-2.4.0/src/data_cleaner/__init__.py

@@ -0,0 +1,86 @@
+"""Universal Data Cleaner — production-oriented tabular cleaning toolkit.
+
+The package is intentionally dependency-light:
+
+- pandas + numpy for data handling
+- scikit-learn base classes for Pipeline / clone compatibility
+- joblib for artifact persistence
+
+The public API is re-exported here so ``import data_cleaner`` keeps working
+exactly as it did when the library shipped as a single module.
+"""
+
+from __future__ import annotations
+
+from ._version import __version__
+from .artifacts import (
+    assert_schema_stable,
+    compare_fingerprints,
+    fingerprint_dataframe,
+)
+from .cleaner import DataCleaner
+from .config import CleanerConfig
+from .contract import ColumnSpec, Contract, DataContract, DriftReport
+from .decisions import CleaningDecision
+from .integration import (
+    DeduplicationReport,
+    JoinDiagnosis,
+    ReferentialIntegrityReport,
+    check_referential_integrity,
+    deduplicate_by_business_key,
+    diagnose_join,
+)
+from .profiling import (
+    assert_frame_clean,
+    dataset_summary,
+    drop_duplicate_rows,
+    profile_dataframe,
+)
+from .validation import (
+    Expectation,
+    ValidationReport,
+    ValidationResult,
+    between,
+    has_dtype,
+    in_set,
+    matches_pattern,
+    not_null,
+    row_count_between,
+    unique,
+    validate,
+)
+
+__all__ = [
+    "CleanerConfig",
+    "CleaningDecision",
+    "ColumnSpec",
+    "Contract",
+    "DataCleaner",
+    "DataContract",
+    "DeduplicationReport",
+    "DriftReport",
+    "Expectation",
+    "JoinDiagnosis",
+    "ReferentialIntegrityReport",
+    "ValidationReport",
+    "ValidationResult",
+    "__version__",
+    "assert_frame_clean",
+    "assert_schema_stable",
+    "between",
+    "check_referential_integrity",
+    "compare_fingerprints",
+    "dataset_summary",
+    "deduplicate_by_business_key",
+    "diagnose_join",
+    "drop_duplicate_rows",
+    "fingerprint_dataframe",
+    "has_dtype",
+    "in_set",
+    "matches_pattern",
+    "not_null",
+    "profile_dataframe",
+    "row_count_between",
+    "unique",
+    "validate",
+]

{universal_data_cleaner-2.2.0 → universal_data_cleaner-2.4.0}/src/data_cleaner/_version.py

@@ -1,4 +1,4 @@
 """Single source of truth for the package version."""
 
-__version__ = "2.2.0"
-_ARTIFACT_VERSION = 1
+__version__ = "2.4.0"
+_ARTIFACT_VERSION = 2