universal-data-cleaner 2.3.0__tar.gz → 2.4.0__tar.gz

{universal_data_cleaner-2.3.0 → universal_data_cleaner-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: universal-data-cleaner
-Version: 2.3.0
+Version: 2.4.0
 Summary: Production-ready, scikit-learn-compatible data cleaning transformer for pandas.
 Author: Haseeb Khan
 License-Expression: MIT
@@ -241,6 +241,110 @@ restored = DataContract.from_json(payload)
 # restored = DataContract.from_yaml(contract.to_yaml())
 ```
 
+## Expectations, joins, and referential integrity
+
+### Expectations and validation reports
+
+Run lightweight expectations against any batch and get a shareable report:
+
+```python
+from data_cleaner import (
+    between,
+    has_dtype,
+    in_set,
+    matches_pattern,
+    not_null,
+    row_count_between,
+    unique,
+    validate,
+)
+
+report = validate(
+    orders_new,
+    [
+        not_null("order_id"),
+        unique("order_id"),
+        in_set("status", ["new", "paid", "shipped"]),
+        between("amount", 0, 100_000),
+        matches_pattern("email", r"[^@]+@[^@]+\.[^@]+"),
+        has_dtype("created_at", "datetime"),
+        row_count_between(min_rows=1),
+    ],
+)
+print(report.to_markdown())
+report.raise_if_failed()
+```
+
+`report.success` is true when no error-severity expectation failed.
+`report.failures` includes warnings too, so use `severity="warning"` for soft
+checks. `has_dtype` accepts broad kinds (`numeric`, `categorical`, `boolean`,
+`datetime`) or an exact pandas dtype such as `float64`.
+
+Contracts can validate a batch directly and return the same report type:
+
+```python
+from data_cleaner import DataContract
+
+contract = DataContract.from_frame(orders_train)
+result = contract.validate(orders_new)
+result.failures.to_markdown()
+```
+
+`Contract` is a shorthand alias for `DataContract`.
+
+### Join diagnostics
+
+```python
+from data_cleaner import diagnose_join
+
+report = diagnose_join(orders, customers, on="customer_id", how="left")
+
+report.safe  # False on cardinality mismatch, left/right row multiplication, or null keys
+report.cardinality  # e.g. "1:m"
+report.row_multiplier  # output rows / left rows
+report.unmatched_left
+report.unmatched_right
+report.warnings
+report.raise_if_unsafe()
+```
+
+`diagnose_join` defaults to `how="left"` and `validate="1:1"`. It reports
+duplicate keys, null keys, unmatched rows, and the actual output row count
+without materializing the full joined frame.
+
+### Referential integrity
+
+```python
+from data_cleaner import check_referential_integrity
+
+report = check_referential_integrity(
+    orders, customers, child_key="customer_id", parent_key="id"
+)
+report.orphan_rows
+report.orphan_keys
+report.raise_if_invalid()
+```
+
+Null child keys are reported but allowed by default (`allow_null_keys=True`).
+Duplicate parent keys make the relationship invalid because the parent key is
+not a unique reference target.
+
+### Business-key deduplication
+
+```python
+from data_cleaner import deduplicate_by_business_key
+
+deduped, report = deduplicate_by_business_key(
+    orders, key="order_id", keep="last", tie_breaker="updated_at"
+)
+report.removed_rows
+report.removed  # the actual rows that were dropped
+```
+
+The returned frame preserves the input row order and index. `tie_breaker`
+prefers non-null values, and null business keys are reported explicitly.
+The input frame is never mutated.
+
 ## Audit trail, manifests, and fingerprints
 
 Every learned or configured decision is recorded during `fit()` and exposed as
@@ -337,7 +441,21 @@ predictions = pipeline.predict(X_test)
 | `CleaningDecision` | One auditable learned or configured decision |
 | `ColumnSpec` | Declarative expectations and semantics for one column |
 | `DataContract` | A set of column specs, with inference and (de)serialization |
+| `Contract` | Shorthand alias for `DataContract` |
+| `DataContract.validate()` | Check a batch and return a `ValidationReport` |
 | `DriftReport` | Findings from a contract check, with a Markdown summary |
+| `Expectation` | One reusable expectation produced by a factory |
+| `validate` | Evaluate expectations against a frame |
+| `not_null`, `unique`, `in_set`, `between` | Expectation factories |
+| `matches_pattern`, `has_dtype` | Expectation factories |
+| `row_count_between` | Frame-level row-count expectation |
+| `ValidationReport` | Pass/fail results, failures table, Markdown, and raise helper |
+| `diagnose_join` | Diagnose key duplication, cardinality, and row multiplication |
+| `JoinDiagnosis` | Join report with multiplier, unmatched counts, and warnings |
+| `check_referential_integrity` | Find orphan child keys and non-unique parent keys |
+| `ReferentialIntegrityReport` | Orphan counts/keys, null keys, and validity |
+| `deduplicate_by_business_key` | Deduplicate by key and report removed rows |
+| `DeduplicationReport` | Removed rows, duplicate-key counts, and tie-breaker details |
 | `DataCleaner.drift_report()` | Drift findings from the last `transform` |
 | `DataCleaner.fitted_contract()` | The contract used by the fitted cleaner |
 | `DataCleaner.decision_log()` | Tidy DataFrame of cleaning decisions |
@@ -381,13 +499,18 @@ predictions = pipeline.predict(X_test)
 │   ├── __init__.py                    # public API re-exports
 │   ├── _version.py                    # version metadata
 │   ├── _roles.py                      # semantic roles and dtype helpers
+│   ├── artifacts.py                   # fingerprints and runtime metadata
 │   ├── contract.py                    # ColumnSpec / DataContract / DriftReport
 │   ├── config.py                      # CleanerConfig
 │   ├── cleaner.py                     # DataCleaner
+│   ├── decisions.py                   # CleaningDecision and decision log
+│   ├── integration.py                 # joins, referential integrity, dedup
 │   ├── profiling.py                   # profiling and validation helpers
+│   ├── validation.py                  # expectations and ValidationReport
 │   └── py.typed                       # PEP 561 marker
 ├── Universal_Data_Cleaner.ipynb       # end-to-end reference workflow
-├── tests/test_data_cleaner.py         # unit and integration tests
+├── tests/test_data_cleaner.py         # core unit and integration tests
+├── tests/test_validation.py           # expectations and integration tests
 ├── pyproject.toml                     # package metadata and dev tooling
 ├── uv.lock                            # reproducible dependency lock
 ├── .pre-commit-config.yaml            # local lint/format/type hooks

{universal_data_cleaner-2.3.0 → universal_data_cleaner-2.4.0}/README.md

@@ -208,6 +208,110 @@ restored = DataContract.from_json(payload)
 # restored = DataContract.from_yaml(contract.to_yaml())
 ```
 
+## Expectations, joins, and referential integrity
+
+### Expectations and validation reports
+
+Run lightweight expectations against any batch and get a shareable report:
+
+```python
+from data_cleaner import (
+    between,
+    has_dtype,
+    in_set,
+    matches_pattern,
+    not_null,
+    row_count_between,
+    unique,
+    validate,
+)
+
+report = validate(
+    orders_new,
+    [
+        not_null("order_id"),
+        unique("order_id"),
+        in_set("status", ["new", "paid", "shipped"]),
+        between("amount", 0, 100_000),
+        matches_pattern("email", r"[^@]+@[^@]+\.[^@]+"),
+        has_dtype("created_at", "datetime"),
+        row_count_between(min_rows=1),
+    ],
+)
+print(report.to_markdown())
+report.raise_if_failed()
+```
+
+`report.success` is true when no error-severity expectation failed.
+`report.failures` includes warnings too, so use `severity="warning"` for soft
+checks. `has_dtype` accepts broad kinds (`numeric`, `categorical`, `boolean`,
+`datetime`) or an exact pandas dtype such as `float64`.
+
+Contracts can validate a batch directly and return the same report type:
+
+```python
+from data_cleaner import DataContract
+
+contract = DataContract.from_frame(orders_train)
+result = contract.validate(orders_new)
+result.failures.to_markdown()
+```
+
+`Contract` is a shorthand alias for `DataContract`.
+
+### Join diagnostics
+
+```python
+from data_cleaner import diagnose_join
+
+report = diagnose_join(orders, customers, on="customer_id", how="left")
+
+report.safe  # False on cardinality mismatch, left/right row multiplication, or null keys
+report.cardinality  # e.g. "1:m"
+report.row_multiplier  # output rows / left rows
+report.unmatched_left
+report.unmatched_right
+report.warnings
+report.raise_if_unsafe()
+```
+
+`diagnose_join` defaults to `how="left"` and `validate="1:1"`. It reports
+duplicate keys, null keys, unmatched rows, and the actual output row count
+without materializing the full joined frame.
+
+### Referential integrity
+
+```python
+from data_cleaner import check_referential_integrity
+
+report = check_referential_integrity(
+    orders, customers, child_key="customer_id", parent_key="id"
+)
+report.orphan_rows
+report.orphan_keys
+report.raise_if_invalid()
+```
+
+Null child keys are reported but allowed by default (`allow_null_keys=True`).
+Duplicate parent keys make the relationship invalid because the parent key is
+not a unique reference target.
+
+### Business-key deduplication
+
+```python
+from data_cleaner import deduplicate_by_business_key
+
+deduped, report = deduplicate_by_business_key(
+    orders, key="order_id", keep="last", tie_breaker="updated_at"
+)
+report.removed_rows
+report.removed  # the actual rows that were dropped
+```
+
+The returned frame preserves the input row order and index. `tie_breaker`
+prefers non-null values, and null business keys are reported explicitly.
+The input frame is never mutated.
+
 ## Audit trail, manifests, and fingerprints
 
 Every learned or configured decision is recorded during `fit()` and exposed as
@@ -304,7 +408,21 @@ predictions = pipeline.predict(X_test)
 | `CleaningDecision` | One auditable learned or configured decision |
 | `ColumnSpec` | Declarative expectations and semantics for one column |
 | `DataContract` | A set of column specs, with inference and (de)serialization |
+| `Contract` | Shorthand alias for `DataContract` |
+| `DataContract.validate()` | Check a batch and return a `ValidationReport` |
 | `DriftReport` | Findings from a contract check, with a Markdown summary |
+| `Expectation` | One reusable expectation produced by a factory |
+| `validate` | Evaluate expectations against a frame |
+| `not_null`, `unique`, `in_set`, `between` | Expectation factories |
+| `matches_pattern`, `has_dtype` | Expectation factories |
+| `row_count_between` | Frame-level row-count expectation |
+| `ValidationReport` | Pass/fail results, failures table, Markdown, and raise helper |
+| `diagnose_join` | Diagnose key duplication, cardinality, and row multiplication |
+| `JoinDiagnosis` | Join report with multiplier, unmatched counts, and warnings |
+| `check_referential_integrity` | Find orphan child keys and non-unique parent keys |
+| `ReferentialIntegrityReport` | Orphan counts/keys, null keys, and validity |
+| `deduplicate_by_business_key` | Deduplicate by key and report removed rows |
+| `DeduplicationReport` | Removed rows, duplicate-key counts, and tie-breaker details |
 | `DataCleaner.drift_report()` | Drift findings from the last `transform` |
 | `DataCleaner.fitted_contract()` | The contract used by the fitted cleaner |
 | `DataCleaner.decision_log()` | Tidy DataFrame of cleaning decisions |
@@ -348,13 +466,18 @@ predictions = pipeline.predict(X_test)
 │   ├── __init__.py                    # public API re-exports
 │   ├── _version.py                    # version metadata
 │   ├── _roles.py                      # semantic roles and dtype helpers
+│   ├── artifacts.py                   # fingerprints and runtime metadata
 │   ├── contract.py                    # ColumnSpec / DataContract / DriftReport
 │   ├── config.py                      # CleanerConfig
 │   ├── cleaner.py                     # DataCleaner
+│   ├── decisions.py                   # CleaningDecision and decision log
+│   ├── integration.py                 # joins, referential integrity, dedup
 │   ├── profiling.py                   # profiling and validation helpers
+│   ├── validation.py                  # expectations and ValidationReport
 │   └── py.typed                       # PEP 561 marker
 ├── Universal_Data_Cleaner.ipynb       # end-to-end reference workflow
-├── tests/test_data_cleaner.py         # unit and integration tests
+├── tests/test_data_cleaner.py         # core unit and integration tests
+├── tests/test_validation.py           # expectations and integration tests
 ├── pyproject.toml                     # package metadata and dev tooling
 ├── uv.lock                            # reproducible dependency lock
 ├── .pre-commit-config.yaml            # local lint/format/type hooks

{universal_data_cleaner-2.3.0 → universal_data_cleaner-2.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "universal-data-cleaner"
-version = "2.3.0"
+version = "2.4.0"
 description = "Production-ready, scikit-learn-compatible data cleaning transformer for pandas."
 readme = "README.md"
 license = "MIT"

{universal_data_cleaner-2.3.0 → universal_data_cleaner-2.4.0}/src/data_cleaner/__init__.py

@@ -20,28 +20,67 @@ from .artifacts import (
 )
 from .cleaner import DataCleaner
 from .config import CleanerConfig
-from .contract import ColumnSpec, DataContract, DriftReport
+from .contract import ColumnSpec, Contract, DataContract, DriftReport
 from .decisions import CleaningDecision
+from .integration import (
+    DeduplicationReport,
+    JoinDiagnosis,
+    ReferentialIntegrityReport,
+    check_referential_integrity,
+    deduplicate_by_business_key,
+    diagnose_join,
+)
 from .profiling import (
     assert_frame_clean,
     dataset_summary,
     drop_duplicate_rows,
     profile_dataframe,
 )
+from .validation import (
+    Expectation,
+    ValidationReport,
+    ValidationResult,
+    between,
+    has_dtype,
+    in_set,
+    matches_pattern,
+    not_null,
+    row_count_between,
+    unique,
+    validate,
+)
 
 __all__ = [
     "CleanerConfig",
     "CleaningDecision",
     "ColumnSpec",
+    "Contract",
     "DataCleaner",
     "DataContract",
+    "DeduplicationReport",
     "DriftReport",
+    "Expectation",
+    "JoinDiagnosis",
+    "ReferentialIntegrityReport",
+    "ValidationReport",
+    "ValidationResult",
     "__version__",
     "assert_frame_clean",
     "assert_schema_stable",
+    "between",
+    "check_referential_integrity",
     "compare_fingerprints",
     "dataset_summary",
+    "deduplicate_by_business_key",
+    "diagnose_join",
     "drop_duplicate_rows",
     "fingerprint_dataframe",
+    "has_dtype",
+    "in_set",
+    "matches_pattern",
+    "not_null",
     "profile_dataframe",
+    "row_count_between",
+    "unique",
+    "validate",
 ]

{universal_data_cleaner-2.3.0 → universal_data_cleaner-2.4.0}/src/data_cleaner/_version.py

@@ -1,4 +1,4 @@
 """Single source of truth for the package version."""
 
-__version__ = "2.3.0"
+__version__ = "2.4.0"
 _ARTIFACT_VERSION = 2

{universal_data_cleaner-2.3.0 → universal_data_cleaner-2.4.0}/src/data_cleaner/contract.py

@@ -16,6 +16,7 @@ from ._roles import (
     _normalize_role,
     _series_kind,
 )
+from .validation import ValidationReport
 
 if TYPE_CHECKING:
     from .config import CleanerConfig
@@ -217,6 +218,20 @@ class DataContract:
             )
         return cls(columns=tuple(specs), strict=strict)
 
+    @classmethod
+    def from_frame(
+        cls,
+        frame: pd.DataFrame,
+        config: CleanerConfig | None = None,
+        strict: bool = True,
+    ) -> DataContract:
+        """Infer a contract from ``frame`` (alias for :meth:`infer_from`)."""
+        return cls.infer_from(frame, config=config, strict=strict)
+
+    def validate(self, frame: pd.DataFrame) -> ValidationReport:
+        """Check ``frame`` against this contract and return a validation report."""
+        return ValidationReport.from_findings(self.check(frame).findings)
+
     @staticmethod
     def _finding(
         spec: ColumnSpec,
@@ -427,3 +442,7 @@ class DataContract:
                 "`pip install pyyaml` or use to_json()/from_json()."
             ) from exc
         return cls.from_dict(yaml.safe_load(text))
+
+
+#: Shorthand alias for :class:`DataContract`.
+Contract = DataContract