reliable-masked-tabular-imputation 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reliable_masked_tabular_imputation-0.1.0/LICENSE +21 -0
- reliable_masked_tabular_imputation-0.1.0/PKG-INFO +93 -0
- reliable_masked_tabular_imputation-0.1.0/README.md +71 -0
- reliable_masked_tabular_imputation-0.1.0/pyproject.toml +30 -0
- reliable_masked_tabular_imputation-0.1.0/setup.cfg +4 -0
- reliable_masked_tabular_imputation-0.1.0/src/reliable_masked_tabular_imputation/__init__.py +10 -0
- reliable_masked_tabular_imputation-0.1.0/src/reliable_masked_tabular_imputation/masking.py +80 -0
- reliable_masked_tabular_imputation-0.1.0/src/reliable_masked_tabular_imputation/metrics.py +77 -0
- reliable_masked_tabular_imputation-0.1.0/src/reliable_masked_tabular_imputation.egg-info/PKG-INFO +93 -0
- reliable_masked_tabular_imputation-0.1.0/src/reliable_masked_tabular_imputation.egg-info/SOURCES.txt +16 -0
- reliable_masked_tabular_imputation-0.1.0/src/reliable_masked_tabular_imputation.egg-info/dependency_links.txt +1 -0
- reliable_masked_tabular_imputation-0.1.0/src/reliable_masked_tabular_imputation.egg-info/requires.txt +9 -0
- reliable_masked_tabular_imputation-0.1.0/src/reliable_masked_tabular_imputation.egg-info/top_level.txt +2 -0
- reliable_masked_tabular_imputation-0.1.0/src/rmti/masking.py +53 -0
- reliable_masked_tabular_imputation-0.1.0/src/rmti/schema.py +9 -0
- reliable_masked_tabular_imputation-0.1.0/tests/test_adversarial_api.py +220 -0
- reliable_masked_tabular_imputation-0.1.0/tests/test_legacy_compat.py +30 -0
- reliable_masked_tabular_imputation-0.1.0/tests/test_public_api.py +54 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mahesh Reddy Pagadala
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: reliable-masked-tabular-imputation
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Reproducible MCAR masking and masked-cell evaluation for pandas tabular data
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Classifier: Programming Language :: Python :: 3
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
10
|
+
Classifier: Topic :: Scientific/Engineering
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: numpy>=1.24
|
|
15
|
+
Requires-Dist: pandas>=2.0
|
|
16
|
+
Provides-Extra: test
|
|
17
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
18
|
+
Provides-Extra: build
|
|
19
|
+
Requires-Dist: build>=1.0; extra == "build"
|
|
20
|
+
Requires-Dist: twine>=5.0; extra == "build"
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# Reliable Masked Tabular Imputation (RMTI)
|
|
24
|
+
|
|
25
|
+
A small, installable Python toolkit for **reproducible MCAR masking** and **masked-cell imputation evaluation** on pandas tables.
|
|
26
|
+
|
|
27
|
+
> **Scope:** The software API below is a utility library, not a validated imputation estimator. The accompanying research programme remains **REVISE — G1A IN PROGRESS; G1B BLOCKED**. R002 full MAR verification and R003 original Adult dataset byte provenance are unresolved. No model fitting, baseline fitting, test-set evaluation or scientific performance claim is authorised by this release.
|
|
28
|
+
|
|
29
|
+
## Install
|
|
30
|
+
|
|
31
|
+
Requires Python 3.10 or later.
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
python -m pip install -e ".[test]"
|
|
35
|
+
python -m pytest
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The package is installed from this repository. **It is not claimed to be published on PyPI.**
|
|
39
|
+
|
|
40
|
+
## Quick start
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
import pandas as pd
|
|
44
|
+
from reliable_masked_tabular_imputation import (
|
|
45
|
+
make_mcar_mask, apply_mask, masked_mae, masked_rmse
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
truth = pd.DataFrame({"age": [20., 30., 40., 50.], "score": [1., 2., 3., 4.]})
|
|
49
|
+
mask = make_mcar_mask(truth, rate=0.25, random_state=42)
|
|
50
|
+
incomplete = apply_mask(truth, mask)
|
|
51
|
+
|
|
52
|
+
# Illustrative baseline only, not an endorsed research experiment:
|
|
53
|
+
filled = incomplete.fillna(incomplete.mean())
|
|
54
|
+
print("MAE:", masked_mae(truth, filled, mask))
|
|
55
|
+
print("RMSE:", masked_rmse(truth, filled, mask))
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### API contracts
|
|
59
|
+
|
|
60
|
+
- `make_mcar_mask(frame, rate, random_state=0, columns=None)` uniformly samples **exactly floor(rate × eligible observed cells)** without replacement. The boolean mask marks *newly* hidden cells only. It never targets originally missing cells. An integer seed repeats the result; a supplied NumPy Generator is consumed. Selection is global, not column-stratified.
|
|
61
|
+
- `apply_mask(frame, mask)` returns a masked copy, preserving the input. The mask must have identical ordered index/columns and boolean entries.
|
|
62
|
+
- `masked_mae(truth, imputed, mask)` and `masked_rmse(...)` score **only True mask cells**. These cells must be finite numeric values in both truth and prediction. For integer-typed selected values with `abs(value) > 2**53`, both raise `ValueError` before conversion to `float64` to avoid silent precision loss. Values exactly equal to `±2**53` are allowed. Selected object-backed values other than Python/NumPy integer, boolean and standard floating-point scalars (including `decimal.Decimal`, `fractions.Fraction`, numeric strings and complex numbers) raise `TypeError` before `float64` conversion. NumPy extended-precision floating scalars wider than 64 bits are unsupported and raise `TypeError`. Only selected cells are validated; unsupported values elsewhere do not affect scoring. Empty masks raise `ValueError`. Other categorical columns are allowed.
|
|
63
|
+
- Unique row and column labels are required for mask generation. A rate of 0 or no eligible cells yields an all-False mask.
|
|
64
|
+
|
|
65
|
+
This library deliberately does **not** label MCAR simulations as MAR, provide a learned imputer, assert non-inferiority, or certify scientific results.
|
|
66
|
+
|
|
67
|
+
## Important evaluation limitations
|
|
68
|
+
|
|
69
|
+
**Global MCAR is not column-wise MCAR.** The fixed count is allocated uniformly across all initially observed eligible cells. Individual columns can receive zero newly masked cells, particularly if few values were initially observed. The returned mask describes *artificially* hidden entries, not the union with existing missing values. If original missingness was non-MCAR, the overall missingness after applying this mask is not MCAR.
|
|
70
|
+
|
|
71
|
+
**Dtype contract:** `apply_mask` never mutates the caller's DataFrame. NumPy-backed integer columns receiving a new masked value are first converted to nullable pandas `Int64` or `UInt64` so that unmasked integers retain their exact values, including `int64`/`uint64` extrema. Nullable integer, float, boolean, string and category columns retain their supported extension dtypes. Categorical values can be masked, but numerical MAE/RMSE require numeric selected cells.
|
|
72
|
+
|
|
73
|
+
**Missingness and nonfinite contract:** `NaN` and pandas `NA` are missing and ineligible for newly generated MCAR masks. `+inf` and `-inf` are considered nonmissing (and hence eligible) by pandas missingness semantics, but selected infinite values are rejected as nonfinite during scoring (`ValueError`).
|
|
74
|
+
|
|
75
|
+
**Metric scope:** `masked_mae` and `masked_rmse` are aggregate errors over selected numeric cells. They do not assess downstream predictive utility, uncertainty, calibration, or statistical validity. Features on different scales may dominate the aggregate; use appropriately scaled inputs or independent per-feature analyses. Numeric overflow during error subtraction raises `ValueError`; scaled reductions avoid unnecessary overflow from squaring or summing large finite errors.
|
|
76
|
+
|
|
77
|
+
**Non-goals:** This package does not itself fit or select imputers. It does not provide validated MAR/MNAR generation, certify research protocols, or establish superiority of any imputation method.
|
|
78
|
+
|
|
79
|
+
## Legacy research modules
|
|
80
|
+
|
|
81
|
+
The distribution also retains the existing experimental `rmti.masking` and `rmti.schema` modules for compatibility with the repository's G1A work. These are **not aliases** of the new public API: legacy `generate_mask` uses per-feature counts and supports an experimental MAR branch; new `make_mcar_mask` uses a global exact-budget MCAR sampler and returns a labelled DataFrame. Do not interchange their masks or assume they have identical sampling or validation contracts. The experimental MAR implementation remains **unverified** and outside the supported v0.1.0 utility API.
|
|
82
|
+
|
|
83
|
+
## Licence
|
|
84
|
+
|
|
85
|
+
MIT License — copyright (c) 2026 Mahesh Reddy Pagadala. See [LICENSE](LICENSE). The licence covers the code distribution; it does not imply scientific validation or approval of the research experiments.
|
|
86
|
+
|
|
87
|
+
## Quality and release
|
|
88
|
+
|
|
89
|
+
Run `python -m pytest`; CI tests Python 3.10–3.12. Before publishing a distribution, run `python -m build` and `python -m twine check dist/*`, verify the wheel in a clean environment, review dependencies and licence, and record release provenance. Do not interpret passing unit tests as a research gate.
|
|
90
|
+
|
|
91
|
+
## Research status
|
|
92
|
+
|
|
93
|
+
R001 was closed by reducing inferential scope. Prespecified non-inferiority procedures failed validation under skewness, so downstream comparisons remain descriptive. R002 (MAR verification) and R003 (original Adult bytes) remain unresolved. Scientific discoveries confirmed: **0**. Retain the project's research decision ledger for the detailed gate record.
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# Reliable Masked Tabular Imputation (RMTI)
|
|
2
|
+
|
|
3
|
+
A small, installable Python toolkit for **reproducible MCAR masking** and **masked-cell imputation evaluation** on pandas tables.
|
|
4
|
+
|
|
5
|
+
> **Scope:** The software API below is a utility library, not a validated imputation estimator. The accompanying research programme remains **REVISE — G1A IN PROGRESS; G1B BLOCKED**. R002 full MAR verification and R003 original Adult dataset byte provenance are unresolved. No model fitting, baseline fitting, test-set evaluation or scientific performance claim is authorised by this release.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
Requires Python 3.10 or later.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
python -m pip install -e ".[test]"
|
|
13
|
+
python -m pytest
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
The package is installed from this repository. **It is not claimed to be published on PyPI.**
|
|
17
|
+
|
|
18
|
+
## Quick start
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
import pandas as pd
|
|
22
|
+
from reliable_masked_tabular_imputation import (
|
|
23
|
+
make_mcar_mask, apply_mask, masked_mae, masked_rmse
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
truth = pd.DataFrame({"age": [20., 30., 40., 50.], "score": [1., 2., 3., 4.]})
|
|
27
|
+
mask = make_mcar_mask(truth, rate=0.25, random_state=42)
|
|
28
|
+
incomplete = apply_mask(truth, mask)
|
|
29
|
+
|
|
30
|
+
# Illustrative baseline only, not an endorsed research experiment:
|
|
31
|
+
filled = incomplete.fillna(incomplete.mean())
|
|
32
|
+
print("MAE:", masked_mae(truth, filled, mask))
|
|
33
|
+
print("RMSE:", masked_rmse(truth, filled, mask))
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
### API contracts
|
|
37
|
+
|
|
38
|
+
- `make_mcar_mask(frame, rate, random_state=0, columns=None)` uniformly samples **exactly floor(rate × eligible observed cells)** without replacement. The boolean mask marks *newly* hidden cells only. It never targets originally missing cells. An integer seed repeats the result; a supplied NumPy Generator is consumed. Selection is global, not column-stratified.
|
|
39
|
+
- `apply_mask(frame, mask)` returns a masked copy, preserving the input. The mask must have identical ordered index/columns and boolean entries.
|
|
40
|
+
- `masked_mae(truth, imputed, mask)` and `masked_rmse(...)` score **only True mask cells**. These cells must be finite numeric values in both truth and prediction. For integer-typed selected values with `abs(value) > 2**53`, both raise `ValueError` before conversion to `float64` to avoid silent precision loss. Values exactly equal to `±2**53` are allowed. Selected object-backed values other than Python/NumPy integer, boolean and standard floating-point scalars (including `decimal.Decimal`, `fractions.Fraction`, numeric strings and complex numbers) raise `TypeError` before `float64` conversion. NumPy extended-precision floating scalars wider than 64 bits are unsupported and raise `TypeError`. Only selected cells are validated; unsupported values elsewhere do not affect scoring. Empty masks raise `ValueError`. Other categorical columns are allowed.
|
|
41
|
+
- Unique row and column labels are required for mask generation. A rate of 0 or no eligible cells yields an all-False mask.
|
|
42
|
+
|
|
43
|
+
This library deliberately does **not** label MCAR simulations as MAR, provide a learned imputer, assert non-inferiority, or certify scientific results.
|
|
44
|
+
|
|
45
|
+
## Important evaluation limitations
|
|
46
|
+
|
|
47
|
+
**Global MCAR is not column-wise MCAR.** The fixed count is allocated uniformly across all initially observed eligible cells. Individual columns can receive zero newly masked cells, particularly if few values were initially observed. The returned mask describes *artificially* hidden entries, not the union with existing missing values. If original missingness was non-MCAR, the overall missingness after applying this mask is not MCAR.
|
|
48
|
+
|
|
49
|
+
**Dtype contract:** `apply_mask` never mutates the caller's DataFrame. NumPy-backed integer columns receiving a new masked value are first converted to nullable pandas `Int64` or `UInt64` so that unmasked integers retain their exact values, including `int64`/`uint64` extrema. Nullable integer, float, boolean, string and category columns retain their supported extension dtypes. Categorical values can be masked, but numerical MAE/RMSE require numeric selected cells.
|
|
50
|
+
|
|
51
|
+
**Missingness and nonfinite contract:** `NaN` and pandas `NA` are missing and ineligible for newly generated MCAR masks. `+inf` and `-inf` are considered nonmissing (and hence eligible) by pandas missingness semantics, but selected infinite values are rejected as nonfinite during scoring (`ValueError`).
|
|
52
|
+
|
|
53
|
+
**Metric scope:** `masked_mae` and `masked_rmse` are aggregate errors over selected numeric cells. They do not assess downstream predictive utility, uncertainty, calibration, or statistical validity. Features on different scales may dominate the aggregate; use appropriately scaled inputs or independent per-feature analyses. Numeric overflow during error subtraction raises `ValueError`; scaled reductions avoid unnecessary overflow from squaring or summing large finite errors.
|
|
54
|
+
|
|
55
|
+
**Non-goals:** This package does not itself fit or select imputers. It does not provide validated MAR/MNAR generation, certify research protocols, or establish superiority of any imputation method.
|
|
56
|
+
|
|
57
|
+
## Legacy research modules
|
|
58
|
+
|
|
59
|
+
The distribution also retains the existing experimental `rmti.masking` and `rmti.schema` modules for compatibility with the repository's G1A work. These are **not aliases** of the new public API: legacy `generate_mask` uses per-feature counts and supports an experimental MAR branch; new `make_mcar_mask` uses a global exact-budget MCAR sampler and returns a labelled DataFrame. Do not interchange their masks or assume they have identical sampling or validation contracts. The experimental MAR implementation remains **unverified** and outside the supported v0.1.0 utility API.
|
|
60
|
+
|
|
61
|
+
## Licence
|
|
62
|
+
|
|
63
|
+
MIT License — copyright (c) 2026 Mahesh Reddy Pagadala. See [LICENSE](LICENSE). The licence covers the code distribution; it does not imply scientific validation or approval of the research experiments.
|
|
64
|
+
|
|
65
|
+
## Quality and release
|
|
66
|
+
|
|
67
|
+
Run `python -m pytest`; CI tests Python 3.10–3.12. Before publishing a distribution, run `python -m build` and `python -m twine check dist/*`, verify the wheel in a clean environment, review dependencies and licence, and record release provenance. Do not interpret passing unit tests as a research gate.
|
|
68
|
+
|
|
69
|
+
## Research status
|
|
70
|
+
|
|
71
|
+
R001 was closed by reducing inferential scope. Prespecified non-inferiority procedures failed validation under skewness, so downstream comparisons remain descriptive. R002 (MAR verification) and R003 (original Adult bytes) remain unresolved. Scientific discoveries confirmed: **0**. Retain the project's research decision ledger for the detailed gate record.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "reliable-masked-tabular-imputation"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Reproducible MCAR masking and masked-cell evaluation for pandas tabular data"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
dependencies = ["numpy>=1.24", "pandas>=2.0"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Programming Language :: Python :: 3.10",
|
|
17
|
+
"Programming Language :: Python :: 3.11",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Topic :: Scientific/Engineering",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
test = ["pytest>=7"]
|
|
24
|
+
build = ["build>=1.0", "twine>=5.0"]
|
|
25
|
+
|
|
26
|
+
[tool.setuptools.packages.find]
|
|
27
|
+
where = ["src"]
|
|
28
|
+
|
|
29
|
+
[tool.pytest.ini_options]
|
|
30
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Reproducible masking and evaluation tools for tabular imputation.
|
|
2
|
+
|
|
3
|
+
The public API currently covers MCAR masking and masked-cell metrics.
|
|
4
|
+
It does not certify MAR simulation or an imputation model.
|
|
5
|
+
"""
|
|
6
|
+
from .masking import apply_mask, make_mcar_mask
|
|
7
|
+
from .metrics import masked_mae, masked_rmse
|
|
8
|
+
|
|
9
|
+
__version__ = "0.1.0"
|
|
10
|
+
__all__ = ["make_mcar_mask", "apply_mask", "masked_mae", "masked_rmse"]
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Deterministic, exact-budget MCAR masking for pandas DataFrames."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from collections.abc import Sequence
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def make_mcar_mask(
|
|
10
|
+
frame: pd.DataFrame,
|
|
11
|
+
rate: float,
|
|
12
|
+
*,
|
|
13
|
+
random_state: int | np.random.Generator = 0,
|
|
14
|
+
columns: Sequence[str] | None = None,
|
|
15
|
+
) -> pd.DataFrame:
|
|
16
|
+
"""Choose exactly floor(rate * eligible_cells) observed cells uniformly.
|
|
17
|
+
|
|
18
|
+
Eligible cells are originally nonmissing cells in selected columns. The
|
|
19
|
+
returned boolean mask is True *only* for newly hidden cells. Selection is
|
|
20
|
+
uniform over eligible cells, not stratified by column; per-column rates
|
|
21
|
+
may differ. The caller's DataFrame is never modified.
|
|
22
|
+
|
|
23
|
+
For repeatability, supply an integer seed. A supplied Generator is
|
|
24
|
+
advanced and therefore is not reset between calls.
|
|
25
|
+
"""
|
|
26
|
+
if not isinstance(frame, pd.DataFrame):
|
|
27
|
+
raise TypeError("frame must be a pandas DataFrame")
|
|
28
|
+
if isinstance(rate, (bool, np.bool_)) or not isinstance(rate, (int, float, np.integer, np.floating)):
|
|
29
|
+
raise TypeError("rate must be a real number in [0, 1]")
|
|
30
|
+
if not np.isfinite(rate) or not 0 <= rate <= 1:
|
|
31
|
+
raise ValueError("rate must be finite and in [0, 1]")
|
|
32
|
+
if not frame.columns.is_unique or not frame.index.is_unique:
|
|
33
|
+
raise ValueError("frame must have unique column and index labels")
|
|
34
|
+
if columns is None:
|
|
35
|
+
positions = list(range(frame.shape[1]))
|
|
36
|
+
else:
|
|
37
|
+
if isinstance(columns, str):
|
|
38
|
+
raise TypeError("columns must be a sequence of names, not a string")
|
|
39
|
+
selected = list(columns)
|
|
40
|
+
if len(selected) != len(set(selected)):
|
|
41
|
+
raise ValueError("columns contains duplicates")
|
|
42
|
+
unknown = [c for c in selected if c not in frame.columns]
|
|
43
|
+
if unknown:
|
|
44
|
+
raise KeyError(f"Unknown columns: {unknown!r}")
|
|
45
|
+
positions = [frame.columns.get_loc(c) for c in selected]
|
|
46
|
+
eligible = np.zeros(frame.shape, dtype=bool)
|
|
47
|
+
if positions:
|
|
48
|
+
eligible[:, positions] = frame.iloc[:, positions].notna().to_numpy()
|
|
49
|
+
candidates = np.flatnonzero(eligible.ravel())
|
|
50
|
+
budget = int(np.floor(float(rate) * len(candidates)))
|
|
51
|
+
generator = random_state if isinstance(random_state, np.random.Generator) else np.random.default_rng(random_state)
|
|
52
|
+
chosen = generator.choice(candidates, size=budget, replace=False)
|
|
53
|
+
mask = np.zeros(frame.shape, dtype=bool)
|
|
54
|
+
mask.ravel()[chosen] = True
|
|
55
|
+
return pd.DataFrame(mask, index=frame.index, columns=frame.columns)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def apply_mask(frame: pd.DataFrame, mask: pd.DataFrame) -> pd.DataFrame:
|
|
59
|
+
"""Return a copy with True mask positions set missing; never mutate inputs.
|
|
60
|
+
|
|
61
|
+
Require identical ordered axes to prevent silent pandas alignment bugs.
|
|
62
|
+
Masked NumPy integer columns are converted to nullable pandas Int64/UInt64
|
|
63
|
+
(preserving unmasked integer values exactly) rather than promoted to float.
|
|
64
|
+
"""
|
|
65
|
+
if not isinstance(frame, pd.DataFrame) or not isinstance(mask, pd.DataFrame):
|
|
66
|
+
raise TypeError("frame and mask must be pandas DataFrames")
|
|
67
|
+
if not frame.index.equals(mask.index) or not frame.columns.equals(mask.columns):
|
|
68
|
+
raise ValueError("mask index and columns must match frame exactly")
|
|
69
|
+
if not all(pd.api.types.is_bool_dtype(dtype) for dtype in mask.dtypes):
|
|
70
|
+
raise TypeError("mask must contain only boolean values")
|
|
71
|
+
if mask.isna().to_numpy().any():
|
|
72
|
+
raise ValueError("mask cannot contain missing values")
|
|
73
|
+
result = frame.copy(deep=True)
|
|
74
|
+
for column in frame.columns:
|
|
75
|
+
dtype = frame[column].dtype
|
|
76
|
+
if (mask[column].to_numpy(dtype=bool).any()
|
|
77
|
+
and isinstance(dtype, np.dtype) and dtype.kind in ("i", "u")):
|
|
78
|
+
nullable_dtype = "Int64" if dtype.kind == "i" else "UInt64"
|
|
79
|
+
result[column] = frame[column].astype(nullable_dtype)
|
|
80
|
+
return result.mask(mask)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Metrics restricted to artificially hidden numeric cells."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _masked_errors(truth: pd.DataFrame, imputed: pd.DataFrame, mask: pd.DataFrame) -> np.ndarray:
|
|
9
|
+
for name, obj in (("truth", truth), ("imputed", imputed), ("mask", mask)):
|
|
10
|
+
if not isinstance(obj, pd.DataFrame):
|
|
11
|
+
raise TypeError(f"{name} must be a pandas DataFrame")
|
|
12
|
+
if not truth.index.equals(imputed.index) or not truth.columns.equals(imputed.columns):
|
|
13
|
+
raise ValueError("truth and imputed axes must match exactly")
|
|
14
|
+
if not truth.index.equals(mask.index) or not truth.columns.equals(mask.columns):
|
|
15
|
+
raise ValueError("mask axes must match truth exactly")
|
|
16
|
+
if not all(pd.api.types.is_bool_dtype(dtype) for dtype in mask.dtypes):
|
|
17
|
+
raise TypeError("mask must be boolean")
|
|
18
|
+
if mask.isna().to_numpy().any():
|
|
19
|
+
raise ValueError("mask cannot contain missing values")
|
|
20
|
+
selected = mask.to_numpy(dtype=bool)
|
|
21
|
+
if not selected.any():
|
|
22
|
+
raise ValueError("mask has no selected cells")
|
|
23
|
+
# Convert only selected cells; unrelated categorical columns may coexist.
|
|
24
|
+
truth_values = truth.to_numpy(dtype=object)[selected]
|
|
25
|
+
pred_values = imputed.to_numpy(dtype=object)[selected]
|
|
26
|
+
# Validate selected scalar types before coercion: Decimal/Fraction and
|
|
27
|
+
# arbitrary object-backed values are outside the float64 scoring contract.
|
|
28
|
+
for value in (*truth_values, *pred_values):
|
|
29
|
+
if not isinstance(value, (bool, np.bool_, int, np.integer, float, np.floating)):
|
|
30
|
+
raise TypeError("selected cells must use supported Python/NumPy numeric scalar types")
|
|
31
|
+
if isinstance(value, np.floating) and np.dtype(type(value)).itemsize > 8:
|
|
32
|
+
raise TypeError("selected extended-precision floats are not supported")
|
|
33
|
+
# float64 represents all integer values with abs(value) <= 2**53.
|
|
34
|
+
if (isinstance(value, (int, np.integer))
|
|
35
|
+
and not isinstance(value, (bool, np.bool_))
|
|
36
|
+
and abs(int(value)) > 2**53):
|
|
37
|
+
raise ValueError("selected integer exceeds float64 exactness boundary (abs(value) > 2**53)")
|
|
38
|
+
try:
|
|
39
|
+
actual = np.asarray(truth_values, dtype=float)
|
|
40
|
+
predicted = np.asarray(pred_values, dtype=float)
|
|
41
|
+
except (ValueError, TypeError) as exc:
|
|
42
|
+
raise TypeError("selected truth and imputed cells must be numeric") from exc
|
|
43
|
+
if not np.isfinite(actual).all() or not np.isfinite(predicted).all():
|
|
44
|
+
raise ValueError("selected cells must contain finite numeric values")
|
|
45
|
+
with np.errstate(over="ignore", invalid="ignore"):
|
|
46
|
+
errors = predicted - actual
|
|
47
|
+
if not np.isfinite(errors).all():
|
|
48
|
+
raise ValueError("numeric overflow when computing masked-cell errors")
|
|
49
|
+
return errors
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def masked_mae(truth: pd.DataFrame, imputed: pd.DataFrame, mask: pd.DataFrame) -> float:
|
|
53
|
+
"""Mean absolute error on hidden numeric cells only.
|
|
54
|
+
|
|
55
|
+
Raise ValueError for selected integer values with abs(value) > 2**53,
|
|
56
|
+
since conversion to float64 may silently lose integer precision.
|
|
57
|
+
"""
|
|
58
|
+
errors = np.abs(_masked_errors(truth, imputed, mask))
|
|
59
|
+
# A mean of finite errors is finite; scaled sum avoids accumulator overflow.
|
|
60
|
+
scale = float(np.max(errors))
|
|
61
|
+
if scale == 0.0:
|
|
62
|
+
return 0.0
|
|
63
|
+
return float(np.mean(errors / scale) * scale)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def masked_rmse(truth: pd.DataFrame, imputed: pd.DataFrame, mask: pd.DataFrame) -> float:
|
|
67
|
+
"""Root mean squared error on hidden numeric cells only.
|
|
68
|
+
|
|
69
|
+
Raise ValueError for selected integer values with abs(value) > 2**53,
|
|
70
|
+
since conversion to float64 may silently lose integer precision.
|
|
71
|
+
"""
|
|
72
|
+
errors = _masked_errors(truth, imputed, mask)
|
|
73
|
+
scale = float(np.max(np.abs(errors)))
|
|
74
|
+
if scale == 0.0:
|
|
75
|
+
return 0.0
|
|
76
|
+
# Scale first so squaring large-but-finite errors cannot overflow.
|
|
77
|
+
return float(np.sqrt(np.mean(np.square(errors / scale))) * scale)
|
reliable_masked_tabular_imputation-0.1.0/src/reliable_masked_tabular_imputation.egg-info/PKG-INFO
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: reliable-masked-tabular-imputation
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Reproducible MCAR masking and masked-cell evaluation for pandas tabular data
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Classifier: Programming Language :: Python :: 3
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
10
|
+
Classifier: Topic :: Scientific/Engineering
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: numpy>=1.24
|
|
15
|
+
Requires-Dist: pandas>=2.0
|
|
16
|
+
Provides-Extra: test
|
|
17
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
18
|
+
Provides-Extra: build
|
|
19
|
+
Requires-Dist: build>=1.0; extra == "build"
|
|
20
|
+
Requires-Dist: twine>=5.0; extra == "build"
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# Reliable Masked Tabular Imputation (RMTI)
|
|
24
|
+
|
|
25
|
+
A small, installable Python toolkit for **reproducible MCAR masking** and **masked-cell imputation evaluation** on pandas tables.
|
|
26
|
+
|
|
27
|
+
> **Scope:** The software API below is a utility library, not a validated imputation estimator. The accompanying research programme remains **REVISE — G1A IN PROGRESS; G1B BLOCKED**. R002 full MAR verification and R003 original Adult dataset byte provenance are unresolved. No model fitting, baseline fitting, test-set evaluation or scientific performance claim is authorised by this release.
|
|
28
|
+
|
|
29
|
+
## Install
|
|
30
|
+
|
|
31
|
+
Requires Python 3.10 or later.
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
python -m pip install -e ".[test]"
|
|
35
|
+
python -m pytest
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The package is installed from this repository. **It is not claimed to be published on PyPI.**
|
|
39
|
+
|
|
40
|
+
## Quick start
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
import pandas as pd
|
|
44
|
+
from reliable_masked_tabular_imputation import (
|
|
45
|
+
make_mcar_mask, apply_mask, masked_mae, masked_rmse
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
truth = pd.DataFrame({"age": [20., 30., 40., 50.], "score": [1., 2., 3., 4.]})
|
|
49
|
+
mask = make_mcar_mask(truth, rate=0.25, random_state=42)
|
|
50
|
+
incomplete = apply_mask(truth, mask)
|
|
51
|
+
|
|
52
|
+
# Illustrative baseline only, not an endorsed research experiment:
|
|
53
|
+
filled = incomplete.fillna(incomplete.mean())
|
|
54
|
+
print("MAE:", masked_mae(truth, filled, mask))
|
|
55
|
+
print("RMSE:", masked_rmse(truth, filled, mask))
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### API contracts
|
|
59
|
+
|
|
60
|
+
- `make_mcar_mask(frame, rate, random_state=0, columns=None)` uniformly samples **exactly floor(rate × eligible observed cells)** without replacement. The boolean mask marks *newly* hidden cells only. It never targets originally missing cells. An integer seed repeats the result; a supplied NumPy Generator is consumed. Selection is global, not column-stratified.
|
|
61
|
+
- `apply_mask(frame, mask)` returns a masked copy, preserving the input. The mask must have identical ordered index/columns and boolean entries.
|
|
62
|
+
- `masked_mae(truth, imputed, mask)` and `masked_rmse(...)` score **only True mask cells**. These cells must be finite numeric values in both truth and prediction. For integer-typed selected values with `abs(value) > 2**53`, both raise `ValueError` before conversion to `float64` to avoid silent precision loss. Values exactly equal to `±2**53` are allowed. Selected object-backed values other than Python/NumPy integer, boolean and standard floating-point scalars (including `decimal.Decimal`, `fractions.Fraction`, numeric strings and complex numbers) raise `TypeError` before `float64` conversion. NumPy extended-precision floating scalars wider than 64 bits are unsupported and raise `TypeError`. Only selected cells are validated; unsupported values elsewhere do not affect scoring. Empty masks raise `ValueError`. Other categorical columns are allowed.
|
|
63
|
+
- Unique row and column labels are required for mask generation. A rate of 0 or no eligible cells yields an all-False mask.
|
|
64
|
+
|
|
65
|
+
This library deliberately does **not** label MCAR simulations as MAR, provide a learned imputer, assert non-inferiority, or certify scientific results.
|
|
66
|
+
|
|
67
|
+
## Important evaluation limitations
|
|
68
|
+
|
|
69
|
+
**Global MCAR is not column-wise MCAR.** The fixed count is allocated uniformly across all initially observed eligible cells. Individual columns can receive zero newly masked cells, particularly if few values were initially observed. The returned mask describes *artificially* hidden entries, not the union with existing missing values. If original missingness was non-MCAR, the overall missingness after applying this mask is not MCAR.
|
|
70
|
+
|
|
71
|
+
**Dtype contract:** `apply_mask` never mutates the caller's DataFrame. NumPy-backed integer columns receiving a new masked value are first converted to nullable pandas `Int64` or `UInt64` so that unmasked integers retain their exact values, including `int64`/`uint64` extrema. Nullable integer, float, boolean, string and category columns retain their supported extension dtypes. Categorical values can be masked, but numerical MAE/RMSE require numeric selected cells.
|
|
72
|
+
|
|
73
|
+
**Missingness and nonfinite contract:** `NaN` and pandas `NA` are missing and ineligible for newly generated MCAR masks. `+inf` and `-inf` are considered nonmissing (and hence eligible) by pandas missingness semantics, but selected infinite values are rejected as nonfinite during scoring (`ValueError`).
|
|
74
|
+
|
|
75
|
+
**Metric scope:** `masked_mae` and `masked_rmse` are aggregate errors over selected numeric cells. They do not assess downstream predictive utility, uncertainty, calibration, or statistical validity. Features on different scales may dominate the aggregate; use appropriately scaled inputs or independent per-feature analyses. Numeric overflow during error subtraction raises `ValueError`; scaled reductions avoid unnecessary overflow from squaring or summing large finite errors.
|
|
76
|
+
|
|
77
|
+
**Non-goals:** This package does not itself fit or select imputers. It does not provide validated MAR/MNAR generation, certify research protocols, or establish superiority of any imputation method.
|
|
78
|
+
|
|
79
|
+
## Legacy research modules
|
|
80
|
+
|
|
81
|
+
The distribution also retains the existing experimental `rmti.masking` and `rmti.schema` modules for compatibility with the repository's G1A work. These are **not aliases** of the new public API: legacy `generate_mask` uses per-feature counts and supports an experimental MAR branch; new `make_mcar_mask` uses a global exact-budget MCAR sampler and returns a labelled DataFrame. Do not interchange their masks or assume they have identical sampling or validation contracts. The experimental MAR implementation remains **unverified** and outside the supported v0.1.0 utility API.
|
|
82
|
+
|
|
83
|
+
## Licence
|
|
84
|
+
|
|
85
|
+
MIT License — copyright (c) 2026 Mahesh Reddy Pagadala. See [LICENSE](LICENSE). The licence covers the code distribution; it does not imply scientific validation or approval of the research experiments.
|
|
86
|
+
|
|
87
|
+
## Quality and release
|
|
88
|
+
|
|
89
|
+
Run `python -m pytest`; CI tests Python 3.10–3.12. Before publishing a distribution, run `python -m build` and `python -m twine check dist/*`, verify the wheel in a clean environment, review dependencies and licence, and record release provenance. Do not interpret passing unit tests as a research gate.
|
|
90
|
+
|
|
91
|
+
## Research status
|
|
92
|
+
|
|
93
|
+
R001 was closed by reducing inferential scope. Prespecified non-inferiority procedures failed validation under skewness, so downstream comparisons remain descriptive. R002 (MAR verification) and R003 (original Adult bytes) remain unresolved. Scientific discoveries confirmed: **0**. Retain the project's research decision ledger for the detailed gate record.
|
reliable_masked_tabular_imputation-0.1.0/src/reliable_masked_tabular_imputation.egg-info/SOURCES.txt
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/reliable_masked_tabular_imputation/__init__.py
|
|
5
|
+
src/reliable_masked_tabular_imputation/masking.py
|
|
6
|
+
src/reliable_masked_tabular_imputation/metrics.py
|
|
7
|
+
src/reliable_masked_tabular_imputation.egg-info/PKG-INFO
|
|
8
|
+
src/reliable_masked_tabular_imputation.egg-info/SOURCES.txt
|
|
9
|
+
src/reliable_masked_tabular_imputation.egg-info/dependency_links.txt
|
|
10
|
+
src/reliable_masked_tabular_imputation.egg-info/requires.txt
|
|
11
|
+
src/reliable_masked_tabular_imputation.egg-info/top_level.txt
|
|
12
|
+
src/rmti/masking.py
|
|
13
|
+
src/rmti/schema.py
|
|
14
|
+
tests/test_adversarial_api.py
|
|
15
|
+
tests/test_legacy_compat.py
|
|
16
|
+
tests/test_public_api.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""G1A fixed-count MCAR/MAR mask generator. No model fitting."""
|
|
2
|
+
import hashlib
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def stable_seed(seed, feature):
|
|
7
|
+
b = f'{int(seed)}|{feature}'.encode()
|
|
8
|
+
return int.from_bytes(hashlib.blake2b(b, digest_size=8).digest(), 'big')
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def probabilities(anchors, rate, slope=1.0, pmin=0.01, pmax=0.99):
|
|
12
|
+
"""Clipped logistic inclusion weights; intercept calibrated to mean(rate)."""
|
|
13
|
+
z = np.clip(np.asarray(anchors, dtype=float), -10, 10)
|
|
14
|
+
if z.ndim != 1 or not np.isfinite(z).all() or not (0 < pmin < rate < pmax < 1):
|
|
15
|
+
raise ValueError('invalid anchors or rate')
|
|
16
|
+
lo, hi = -100.0, 100.0
|
|
17
|
+
for _ in range(120):
|
|
18
|
+
mid = (lo + hi)/2
|
|
19
|
+
p = np.clip(1/(1+np.exp(-np.clip(mid+slope*z, -700, 700))),pmin,pmax)
|
|
20
|
+
if p.mean() < rate: lo = mid
|
|
21
|
+
else: hi = mid
|
|
22
|
+
return np.clip(1/(1+np.exp(-np.clip((lo+hi)/2+slope*z,-700,700))),pmin,pmax)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def generate_mask(frame, features, mechanism, rate, seed, anchor_data=None, eligible=None):
|
|
26
|
+
"""Higher weighted MAR selection is by Gumbel-top-k; per-feature exact counts."""
|
|
27
|
+
if mechanism not in ('MCAR','MAR') or not 0 < rate < 1:
|
|
28
|
+
raise ValueError('invalid mechanism or rate')
|
|
29
|
+
features = list(features)
|
|
30
|
+
if not features or len(set(features)) != len(features) or not set(features).issubset(frame.columns):
|
|
31
|
+
raise ValueError('invalid features')
|
|
32
|
+
if mechanism == 'MAR':
|
|
33
|
+
if anchor_data is None or set(anchor_data.columns) & set(features) or len(anchor_data) != len(frame):
|
|
34
|
+
raise ValueError('MAR requires exclusively unmasked anchor columns')
|
|
35
|
+
if anchor_data.shape[1] != 1:
|
|
36
|
+
raise ValueError('G1A requires one pre-standardized anchor')
|
|
37
|
+
z = anchor_data.iloc[:,0].to_numpy(dtype=float)
|
|
38
|
+
n = len(frame)
|
|
39
|
+
allow = np.ones((n,len(features)),dtype=bool) if eligible is None else np.asarray(eligible,dtype=bool)
|
|
40
|
+
if allow.shape != (n,len(features)): raise ValueError('eligible shape mismatch')
|
|
41
|
+
out = np.zeros_like(allow)
|
|
42
|
+
for j,feature in enumerate(features):
|
|
43
|
+
ids = np.flatnonzero(allow[:,j]); k = int(np.floor(rate*len(ids)+0.5))
|
|
44
|
+
if not k: continue
|
|
45
|
+
rng = np.random.default_rng(stable_seed(seed,feature))
|
|
46
|
+
u = np.clip(rng.random(len(ids)),np.finfo(float).tiny,1-np.finfo(float).eps)
|
|
47
|
+
g = -np.log(-np.log(u))
|
|
48
|
+
if mechanism == 'MCAR': scores = g
|
|
49
|
+
else:
|
|
50
|
+
p = probabilities(z[ids],rate)
|
|
51
|
+
scores = np.log(p) + g
|
|
52
|
+
out[ids[np.argpartition(scores,-k)[-k:]],j] = True
|
|
53
|
+
return out
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""G1A schema validation only. No prediction model."""
|
|
2
|
+
FIELDS = ('dataset','split','split_seed','mask_seed','model_seed','mechanism','level','feature','row_id','cell_id','true_value','predicted_value','uncertainty_score','interval_lower','interval_upper','accepted_flag','review_rank')
|
|
3
|
+
def validate(records):
|
|
4
|
+
for record in records:
|
|
5
|
+
if set(record) != set(FIELDS): raise ValueError('schema mismatch')
|
|
6
|
+
if record['split'] not in ('train','validation','test'): raise ValueError('split')
|
|
7
|
+
if record['mechanism'] not in ('MCAR','MAR','MNAR_STRESS'): raise ValueError('mechanism')
|
|
8
|
+
if not isinstance(record['accepted_flag'],bool): raise ValueError('accepted_flag')
|
|
9
|
+
return True
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""Adversarial public API tests derived from conditional external review C1-C5."""
|
|
2
|
+
import numpy as np
|
|
3
|
+
from decimal import Decimal
|
|
4
|
+
from fractions import Fraction
|
|
5
|
+
import pandas as pd
|
|
6
|
+
import pytest
|
|
7
|
+
from reliable_masked_tabular_imputation import make_mcar_mask, apply_mask, masked_mae, masked_rmse
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_global_budget_and_original_nulls():
|
|
11
|
+
frame = pd.DataFrame({"a": [1., 2., 3., 4.], "b": [np.nan, np.nan, np.nan, 5.]})
|
|
12
|
+
mask = make_mcar_mask(frame, 0.5, random_state=0)
|
|
13
|
+
assert int(mask.to_numpy().sum()) == 2
|
|
14
|
+
assert not mask.iloc[:3, 1].any()
|
|
15
|
+
assert mask.shape == frame.shape
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_seeded_sampling_invariants_across_sizes():
|
|
19
|
+
for rows in (0, 1, 4, 23):
|
|
20
|
+
frame = pd.DataFrame({"x": np.arange(rows, dtype=float),
|
|
21
|
+
"y": np.arange(rows, dtype=float)})
|
|
22
|
+
if rows:
|
|
23
|
+
frame.loc[0, "y"] = np.nan
|
|
24
|
+
for rate in (0, .1, .5, 1):
|
|
25
|
+
mask = make_mcar_mask(frame, rate, random_state=91)
|
|
26
|
+
eligible = frame.notna().to_numpy()
|
|
27
|
+
assert int(mask.to_numpy().sum()) == int(np.floor(rate * eligible.sum()))
|
|
28
|
+
assert not np.logical_and(mask.to_numpy(), ~eligible).any()
|
|
29
|
+
assert mask.equals(make_mcar_mask(frame, rate, random_state=91))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@pytest.mark.parametrize("dtype,values", [
|
|
33
|
+
("Int64", [1, 2, 3]),
|
|
34
|
+
("Float64", [1., 2., 3.]),
|
|
35
|
+
("boolean", [True, False, True]),
|
|
36
|
+
("string", ["a", "b", "c"]),
|
|
37
|
+
("category", ["a", "b", "a"]),
|
|
38
|
+
])
|
|
39
|
+
def test_nullable_and_categorical_dtype_preservation(dtype, values):
|
|
40
|
+
frame = pd.DataFrame({"x": pd.Series(values, dtype=dtype)})
|
|
41
|
+
mask = pd.DataFrame({"x": [False, True, False]})
|
|
42
|
+
out = apply_mask(frame, mask)
|
|
43
|
+
assert str(out.dtypes["x"]) == str(frame.dtypes["x"])
|
|
44
|
+
assert bool(pd.isna(out.loc[1, "x"]))
|
|
45
|
+
assert not pd.isna(frame.loc[1, "x"])
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_plain_int_dtype_converts_to_nullable_without_precision_loss():
|
|
49
|
+
frame = pd.DataFrame({"x": [1, 2, 3]})
|
|
50
|
+
mask = pd.DataFrame({"x": [False, True, False]})
|
|
51
|
+
out = apply_mask(frame, mask)
|
|
52
|
+
assert str(out["x"].dtype) == "Int64"
|
|
53
|
+
assert pd.isna(out.loc[1, "x"])
|
|
54
|
+
assert frame["x"].tolist() == [1, 2, 3]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_multiindex_axes_and_tuple_columns():
|
|
58
|
+
index = pd.MultiIndex.from_tuples([("a", 1), ("a", 2), ("b", 1)])
|
|
59
|
+
columns = pd.MultiIndex.from_tuples([("x", "one"), ("x", "two")])
|
|
60
|
+
frame = pd.DataFrame([[1., 2.], [3., 4.], [5., 6.]], index=index, columns=columns)
|
|
61
|
+
mask = make_mcar_mask(frame, 0.5, random_state=4)
|
|
62
|
+
assert mask.index.equals(frame.index)
|
|
63
|
+
assert mask.columns.equals(frame.columns)
|
|
64
|
+
assert int(apply_mask(frame, mask).isna().to_numpy().sum()) == 3
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_invalid_or_missing_selected_values():
|
|
68
|
+
truth = pd.DataFrame({"x": [1.0, np.nan]})
|
|
69
|
+
pred = pd.DataFrame({"x": [0.0, 0.0]})
|
|
70
|
+
mask = pd.DataFrame({"x": [False, True]})
|
|
71
|
+
with pytest.raises(ValueError, match="finite"):
|
|
72
|
+
masked_mae(truth, pred, mask)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_overflow_on_subtraction_rejected():
|
|
76
|
+
truth = pd.DataFrame({"x": [1e308]})
|
|
77
|
+
pred = pd.DataFrame({"x": [-1e308]})
|
|
78
|
+
mask = pd.DataFrame({"x": [True]})
|
|
79
|
+
with pytest.raises(ValueError, match="overflow"):
|
|
80
|
+
masked_mae(truth, pred, mask)
|
|
81
|
+
with pytest.raises(ValueError, match="overflow"):
|
|
82
|
+
masked_rmse(truth, pred, mask)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_large_finite_errors_do_not_spuriously_overflow_rmse():
|
|
86
|
+
truth = pd.DataFrame({"x": [0., 0.]})
|
|
87
|
+
pred = pd.DataFrame({"x": [1e200, 1e200]})
|
|
88
|
+
mask = pd.DataFrame({"x": [True, True]})
|
|
89
|
+
assert masked_rmse(truth, pred, mask) == pytest.approx(1e200)
|
|
90
|
+
assert masked_mae(truth, pred, mask) == pytest.approx(1e200)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_missing_boolean_mask_rejected():
|
|
94
|
+
frame = pd.DataFrame({"x": [1., 2.]})
|
|
95
|
+
mask = pd.DataFrame({"x": pd.Series([True, pd.NA], dtype="boolean")})
|
|
96
|
+
with pytest.raises(ValueError, match="missing"):
|
|
97
|
+
apply_mask(frame, mask)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@pytest.mark.parametrize("dtype,values,expected_dtype", [
|
|
101
|
+
("int64", [2**63 - 1, 2**63 - 2, 2**63 - 3], "Int64"),
|
|
102
|
+
("uint64", [2**64 - 1, 2**64 - 2, 2**64 - 3], "UInt64"),
|
|
103
|
+
("Int64", [2**63 - 1, 2**63 - 2, 2**63 - 3], "Int64"),
|
|
104
|
+
("UInt64", [2**64 - 1, 2**64 - 2, 2**64 - 3], "UInt64"),
|
|
105
|
+
])
|
|
106
|
+
def test_masking_preserves_unmasked_integer_boundaries(dtype, values, expected_dtype):
|
|
107
|
+
frame = pd.DataFrame({"x": pd.Series(values, dtype=dtype), "note": ["a", "b", "c"]})
|
|
108
|
+
original = frame.copy(deep=True)
|
|
109
|
+
mask = pd.DataFrame({"x": [False, False, True], "note": [False] * 3})
|
|
110
|
+
out = apply_mask(frame, mask)
|
|
111
|
+
assert str(out["x"].dtype) == expected_dtype
|
|
112
|
+
assert int(out.loc[0, "x"]) == values[0]
|
|
113
|
+
assert int(out.loc[1, "x"]) == values[1]
|
|
114
|
+
assert pd.isna(out.loc[2, "x"])
|
|
115
|
+
assert out["note"].tolist() == ["a", "b", "c"]
|
|
116
|
+
pd.testing.assert_frame_equal(frame, original)
|
|
117
|
+
pd.testing.assert_frame_equal(mask, pd.DataFrame({"x": [False, False, True], "note": [False] * 3}))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@pytest.mark.parametrize("dtype,values", [
|
|
121
|
+
("int64", [2**63 - 1, 2**63 - 2]),
|
|
122
|
+
("uint64", [2**64 - 1, 2**64 - 2]),
|
|
123
|
+
("Int64", [2**63 - 1, 2**63 - 2]),
|
|
124
|
+
("UInt64", [2**64 - 1, 2**64 - 2]),
|
|
125
|
+
])
|
|
126
|
+
def test_scoring_rejects_integer_boundary_precision_loss(dtype, values):
|
|
127
|
+
actual = pd.DataFrame({"x": pd.Series([values[0]], dtype=dtype)})
|
|
128
|
+
predicted = pd.DataFrame({"x": pd.Series([values[1]], dtype=dtype)})
|
|
129
|
+
mask = pd.DataFrame({"x": [True]})
|
|
130
|
+
before_actual, before_pred = actual.copy(deep=True), predicted.copy(deep=True)
|
|
131
|
+
for scorer in (masked_mae, masked_rmse):
|
|
132
|
+
with pytest.raises(ValueError, match="exactness boundary"):
|
|
133
|
+
scorer(actual, predicted, mask)
|
|
134
|
+
pd.testing.assert_frame_equal(actual, before_actual)
|
|
135
|
+
pd.testing.assert_frame_equal(predicted, before_pred)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@pytest.mark.parametrize("actual,predicted,raises", [
|
|
139
|
+
(2**53, 2**53 - 1, False),
|
|
140
|
+
(-(2**53), -(2**53) + 1, False),
|
|
141
|
+
(2**53 + 1, 2**53, True),
|
|
142
|
+
(-(2**53) - 1, -(2**53), True),
|
|
143
|
+
])
|
|
144
|
+
def test_metric_integer_exactness_boundary(actual, predicted, raises):
|
|
145
|
+
truth = pd.DataFrame({"x": pd.Series([actual], dtype=object)})
|
|
146
|
+
estimate = pd.DataFrame({"x": pd.Series([predicted], dtype=object)})
|
|
147
|
+
mask = pd.DataFrame({"x": [True]})
|
|
148
|
+
for scorer in (masked_mae, masked_rmse):
|
|
149
|
+
if raises:
|
|
150
|
+
with pytest.raises(ValueError, match="exactness boundary"):
|
|
151
|
+
scorer(truth, estimate, mask)
|
|
152
|
+
else:
|
|
153
|
+
assert scorer(truth, estimate, mask) == pytest.approx(1)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def test_nonfinite_eligibility_and_scoring_policy():
|
|
157
|
+
frame = pd.DataFrame({"x": pd.Series([np.nan, np.inf, -np.inf, 3.0], dtype="Float64"),
|
|
158
|
+
"y": pd.Series([pd.NA, 2.0, 3.0, 4.0], dtype="Float64")})
|
|
159
|
+
mask = make_mcar_mask(frame, 1.0, random_state=10)
|
|
160
|
+
assert mask["x"].tolist() == [False, True, True, True]
|
|
161
|
+
assert mask["y"].tolist() == [False, True, True, True]
|
|
162
|
+
for scorer in (masked_mae, masked_rmse):
|
|
163
|
+
with pytest.raises(ValueError, match="finite"):
|
|
164
|
+
scorer(frame[["x"]], frame[["x"]], mask[["x"]])
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def test_mixed_columns_keep_nonselected_dtypes_and_input_immutability():
|
|
168
|
+
frame = pd.DataFrame({
|
|
169
|
+
"big": pd.Series([2**63 - 1, 2**63 - 2], dtype="int64"),
|
|
170
|
+
"category": pd.Categorical(["a", "b"]),
|
|
171
|
+
"nullable": pd.Series([1, pd.NA], dtype="Int64"),
|
|
172
|
+
})
|
|
173
|
+
before = frame.copy(deep=True)
|
|
174
|
+
mask = pd.DataFrame({"big": [True, False], "category": [False, False], "nullable": [False, False]})
|
|
175
|
+
out = apply_mask(frame, mask)
|
|
176
|
+
assert int(out.loc[1, "big"]) == 2**63 - 2
|
|
177
|
+
assert str(out["big"].dtype) == "Int64"
|
|
178
|
+
assert str(out["nullable"].dtype) == "Int64"
|
|
179
|
+
assert str(out["category"].dtype) == "category"
|
|
180
|
+
pd.testing.assert_frame_equal(frame, before)
|
|
181
|
+
|
|
182
|
+
@pytest.mark.parametrize("unsupported", [
|
|
183
|
+
Decimal("0.1"),
|
|
184
|
+
Fraction(1, 3),
|
|
185
|
+
Decimal("1e1000"),
|
|
186
|
+
"2.5",
|
|
187
|
+
complex(1, 0),
|
|
188
|
+
])
|
|
189
|
+
@pytest.mark.parametrize("invalid_side", ["truth", "imputed"])
|
|
190
|
+
def test_unsupported_object_numeric_selected_values_rejected(unsupported, invalid_side):
|
|
191
|
+
truth = pd.DataFrame({"x": pd.Series([1.0, 2.0], dtype=object),
|
|
192
|
+
"unselected": [Decimal("1.0"), None]})
|
|
193
|
+
predicted = truth.copy(deep=True)
|
|
194
|
+
target = truth if invalid_side == "truth" else predicted
|
|
195
|
+
target.at[0, "x"] = unsupported
|
|
196
|
+
mask = pd.DataFrame({"x": [True, False], "unselected": [False, False]})
|
|
197
|
+
for scorer in (masked_mae, masked_rmse):
|
|
198
|
+
with pytest.raises(TypeError, match="supported Python/NumPy numeric scalar"):
|
|
199
|
+
scorer(truth, predicted, mask)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def test_unselected_unsupported_object_values_do_not_block_scoring():
|
|
203
|
+
truth = pd.DataFrame({"x": pd.Series([3.0, Fraction(1, 3), None], dtype=object),
|
|
204
|
+
"note": [Decimal("0.1"), "other", None]})
|
|
205
|
+
predicted = truth.copy(deep=True)
|
|
206
|
+
predicted.at[0, "x"] = 4.0
|
|
207
|
+
mask = pd.DataFrame({"x": [True, False, False], "note": [False, False, False]})
|
|
208
|
+
before_truth, before_predicted = truth.copy(deep=True), predicted.copy(deep=True)
|
|
209
|
+
assert masked_mae(truth, predicted, mask) == pytest.approx(1.0)
|
|
210
|
+
assert masked_rmse(truth, predicted, mask) == pytest.approx(1.0)
|
|
211
|
+
pd.testing.assert_frame_equal(truth, before_truth)
|
|
212
|
+
pd.testing.assert_frame_equal(predicted, before_predicted)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def test_mixed_object_supported_python_numpy_numeric_scalars():
|
|
216
|
+
truth = pd.DataFrame({"x": pd.Series([3, np.float64(1.5)], dtype=object)})
|
|
217
|
+
predicted = pd.DataFrame({"x": pd.Series([4.0, np.int64(2)], dtype=object)})
|
|
218
|
+
mask = pd.DataFrame({"x": [True, True]})
|
|
219
|
+
assert masked_mae(truth, predicted, mask) == pytest.approx(0.75)
|
|
220
|
+
assert masked_rmse(truth, predicted, mask) == pytest.approx(np.sqrt(0.625))
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Compatibility checks for the pre-existing experimental namespace.
|
|
2
|
+
|
|
3
|
+
These imports and sanity tests do not validate MAR research claims.
|
|
4
|
+
"""
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from rmti.masking import generate_mask, stable_seed
|
|
9
|
+
from rmti.schema import FIELDS, validate
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_legacy_mcar_repeatable_fixed_count():
|
|
13
|
+
frame = pd.DataFrame({"x": np.arange(10)})
|
|
14
|
+
m1 = generate_mask(frame, ["x"], "MCAR", rate=0.3, seed=7)
|
|
15
|
+
m2 = generate_mask(frame, ["x"], "MCAR", rate=0.3, seed=7)
|
|
16
|
+
assert m1.shape == (10, 1)
|
|
17
|
+
assert m1.dtype == np.dtype("bool")
|
|
18
|
+
assert m1.sum() == 3
|
|
19
|
+
assert np.array_equal(m1, m2)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_legacy_seed_determinism():
|
|
23
|
+
assert stable_seed(5, "x") == stable_seed(5, "x")
|
|
24
|
+
assert stable_seed(5, "x") != stable_seed(5, "y")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_legacy_schema_contract():
|
|
28
|
+
record = {name: None for name in FIELDS}
|
|
29
|
+
record.update(split="train", mechanism="MCAR", accepted_flag=False)
|
|
30
|
+
assert validate([record]) is True
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from reliable_masked_tabular_imputation import (
|
|
6
|
+
apply_mask,
|
|
7
|
+
make_mcar_mask,
|
|
8
|
+
masked_mae,
|
|
9
|
+
masked_rmse,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_exact_budget_repeatability_and_no_mutation():
|
|
14
|
+
frame = pd.DataFrame({"a": range(10), "b": range(10, 20)})
|
|
15
|
+
original = frame.copy(deep=True)
|
|
16
|
+
first = make_mcar_mask(frame, 0.35, random_state=123)
|
|
17
|
+
second = make_mcar_mask(frame, 0.35, random_state=123)
|
|
18
|
+
assert first.equals(second)
|
|
19
|
+
assert first.to_numpy().sum() == 7
|
|
20
|
+
assert frame.equals(original)
|
|
21
|
+
masked = apply_mask(frame, first)
|
|
22
|
+
assert masked.isna().to_numpy().sum() == 7
|
|
23
|
+
assert frame.equals(original)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_preexisting_nulls_and_column_selection():
|
|
27
|
+
frame = pd.DataFrame({"a": [1, np.nan, 3, 4], "b": [10, 11, 12, 13]})
|
|
28
|
+
mask = make_mcar_mask(frame, 1, columns=["a"])
|
|
29
|
+
assert mask["a"].tolist() == [True, False, True, True]
|
|
30
|
+
assert not mask["b"].any()
|
|
31
|
+
assert apply_mask(frame, mask).isna().to_numpy().sum() == 4
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_metrics_only_evaluate_hidden_cells():
|
|
35
|
+
truth = pd.DataFrame({"x": [1, 2, 3], "note": ["a", "b", "c"]})
|
|
36
|
+
predicted = pd.DataFrame({"x": [2, 4, 999], "note": ["z", "z", "z"]})
|
|
37
|
+
mask = pd.DataFrame({"x": [True, True, False], "note": [False] * 3})
|
|
38
|
+
assert masked_mae(truth, predicted, mask) == pytest.approx(1.5)
|
|
39
|
+
assert masked_rmse(truth, predicted, mask) == pytest.approx(np.sqrt(2.5))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_invalid_inputs_fail_loudly():
|
|
43
|
+
frame = pd.DataFrame({"x": [1, 2, 3]})
|
|
44
|
+
for rate in [-0.1, 1.1, float("nan")]:
|
|
45
|
+
with pytest.raises(ValueError):
|
|
46
|
+
make_mcar_mask(frame, rate)
|
|
47
|
+
with pytest.raises(KeyError):
|
|
48
|
+
make_mcar_mask(frame, 0.5, columns=["absent"])
|
|
49
|
+
with pytest.raises(ValueError):
|
|
50
|
+
apply_mask(frame, pd.DataFrame({"other": [True, False, False]}))
|
|
51
|
+
with pytest.raises(ValueError):
|
|
52
|
+
masked_mae(frame, frame, pd.DataFrame({"x": [False] * 3}))
|
|
53
|
+
with pytest.raises(ValueError):
|
|
54
|
+
masked_rmse(frame, frame, pd.DataFrame({"x": [True, False]}, index=[0, 1]))
|