mimic-tabular 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
mimic/__init__.py ADDED
@@ -0,0 +1,34 @@
1
+ """MIMIC public API."""
2
+
3
+ from ._version import __version__
4
+ from .decoders import ForestConditionalSampler, IdentityDecoder, LinearMixedFeatureDecoder, MixedFeatureDecoder, NeuralConditionalSampler
5
+ from .datasets import load_paper_dataset, paper_dataset_columns, paper_dataset_names, paper_dataset_registry
6
+ from .encoders import IdentityEncoder, RandomForestPathEncoder, ResNetEncoder, SharedFeatureGroup, SharedResNetEncoder
7
+ from .iterated import IteratedMIMIC
8
+ from .mimic import MIMIC, NearestNeighborPrivacyFilter, mimic_data, sample, sample_dataframe
9
+ from .policies import GenerationPolicy
10
+
11
+ __all__ = [
12
+ "__version__",
13
+ "GenerationPolicy",
14
+ "ForestConditionalSampler",
15
+ "IdentityDecoder",
16
+ "IdentityEncoder",
17
+ "IteratedMIMIC",
18
+ "LinearMixedFeatureDecoder",
19
+ "MIMIC",
20
+ "MixedFeatureDecoder",
21
+ "NeuralConditionalSampler",
22
+ "NearestNeighborPrivacyFilter",
23
+ "load_paper_dataset",
24
+ "paper_dataset_columns",
25
+ "paper_dataset_names",
26
+ "paper_dataset_registry",
27
+ "RandomForestPathEncoder",
28
+ "ResNetEncoder",
29
+ "SharedFeatureGroup",
30
+ "SharedResNetEncoder",
31
+ "mimic_data",
32
+ "sample",
33
+ "sample_dataframe",
34
+ ]
mimic/_torch_utils.py ADDED
@@ -0,0 +1,23 @@
1
+ """Utilities for keeping optional PyTorch state local to model fitting."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from functools import wraps
6
+
7
+ try:
8
+ import torch
9
+ except Exception: # pragma: no cover - import availability is environment dependent
10
+ torch = None
11
+
12
+
13
+ def preserve_torch_rng(method):
14
+ """Restore PyTorch's process-wide RNG state after seeded model fitting."""
15
+
16
+ @wraps(method)
17
+ def wrapped(self, *args, **kwargs):
18
+ if torch is None or getattr(self, "random_state", None) is None:
19
+ return method(self, *args, **kwargs)
20
+ with torch.random.fork_rng(devices=None):
21
+ return method(self, *args, **kwargs)
22
+
23
+ return wrapped
mimic/_version.py ADDED
@@ -0,0 +1,3 @@
1
+ """Canonical MIMIC package version."""
2
+
3
+ __version__ = "0.1.3"
mimic/cli.py ADDED
@@ -0,0 +1,164 @@
1
+ """Command-line interface for one-shot MIMIC data generation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ from pathlib import Path
8
+
9
+ from ._version import __version__
10
+ from .mimic import mimic_data
11
+
12
+
13
+ def main(argv: list[str] | None = None) -> int:
14
+ parser = _build_parser()
15
+ args = parser.parse_args(argv)
16
+
17
+ input_path = Path(args.filename)
18
+ if not input_path.exists():
19
+ parser.error(f"input file does not exist: {input_path}")
20
+
21
+ columns = _parse_columns(args.columns, parser)
22
+ try:
23
+ df = _read_frame(input_path)
24
+ except ValueError as exc:
25
+ parser.error(str(exc))
26
+ try:
27
+ synthetic = mimic_data(
28
+ df,
29
+ columns=columns,
30
+ mode=args.mode,
31
+ capacity=args.capacity,
32
+ save_model=args.save_model,
33
+ load_model=args.load_model,
34
+ refit=args.refit,
35
+ random_state=args.random_state,
36
+ )
37
+ except (OSError, ValueError) as exc:
38
+ parser.error(str(exc))
39
+
40
+ output_path = Path(args.output) if args.output is not None else _default_output_path(input_path)
41
+ try:
42
+ _write_frame(synthetic, output_path)
43
+ except ValueError as exc:
44
+ parser.error(str(exc))
45
+ print(f"Wrote {len(synthetic)} rows to {output_path}")
46
+ return 0
47
+
48
+
49
+ def _build_parser() -> argparse.ArgumentParser:
50
+ parser = argparse.ArgumentParser(
51
+ prog="mimic-data",
52
+ description=(
53
+ "Fit MIMIC on a tabular data file and write a synthetic dataset with "
54
+ "the same number of rows. CSV, Parquet, and Excel files are supported."
55
+ ),
56
+ )
57
+ parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
58
+ parser.add_argument("filename", help="Input .csv, .parquet, .xls, or .xlsx file.")
59
+ parser.add_argument(
60
+ "-o",
61
+ "--output",
62
+ help="Output filename. Defaults to '<input_stem>_mimic<input_suffix>' in the same directory.",
63
+ )
64
+ parser.add_argument(
65
+ "--columns",
66
+ default="auto",
67
+ help=(
68
+ "Column role specification. Use 'auto' for heuristic inference, or pass a JSON mapping "
69
+ "such as '{\"ignore\":[\"id\"],\"regression\":[\"age\"],\"classification\":[\"segment\"]}'. "
70
+ "Prefix with @ to read the JSON mapping from a file."
71
+ ),
72
+ )
73
+ parser.add_argument(
74
+ "--mode",
75
+ default="factorised",
76
+ choices=["identity", "direct", "factorised", "joint", "0", "1", "2", "3"],
77
+ help="MIMIC preset mode. Defaults to %(default)s.",
78
+ )
79
+ parser.add_argument(
80
+ "--capacity",
81
+ type=float,
82
+ default=0.25,
83
+ help="Model capacity in [0, 1]. Defaults to %(default)s.",
84
+ )
85
+ parser.add_argument(
86
+ "--random-state",
87
+ type=int,
88
+ default=None,
89
+ help="Optional random seed for reproducible generation.",
90
+ )
91
+ parser.add_argument(
92
+ "--save-model",
93
+ default=None,
94
+ help="Optional path where the fitted MIMIC model should be saved.",
95
+ )
96
+ parser.add_argument(
97
+ "--load-model",
98
+ default=None,
99
+ help="Optional path to a fitted MIMIC model to load before sampling.",
100
+ )
101
+ parser.add_argument(
102
+ "--refit",
103
+ action="store_true",
104
+ help="Fit a fresh model even when --load-model is supplied.",
105
+ )
106
+ return parser
107
+
108
+
109
+ def _parse_columns(value: str, parser: argparse.ArgumentParser):
110
+ if value in {"auto", "None", "none"}:
111
+ return "auto" if value == "auto" else None
112
+ try:
113
+ if value.startswith("@"):
114
+ text = Path(value[1:]).read_text()
115
+ else:
116
+ text = value
117
+ parsed = json.loads(text)
118
+ except OSError as exc:
119
+ parser.error(f"could not read columns file {value!r}: {exc}")
120
+ except json.JSONDecodeError as exc:
121
+ parser.error(f"--columns must be 'auto' or valid JSON: {exc}")
122
+ if not isinstance(parsed, dict):
123
+ parser.error("--columns JSON must be an object mapping roles to column lists")
124
+ return parsed
125
+
126
+
127
+ def _read_frame(path: Path):
128
+ suffix = path.suffix.lower()
129
+ if suffix == ".csv":
130
+ import pandas as pd
131
+
132
+ return pd.read_csv(path)
133
+ if suffix == ".parquet":
134
+ import pandas as pd
135
+
136
+ return pd.read_parquet(path)
137
+ if suffix in {".xls", ".xlsx"}:
138
+ import pandas as pd
139
+
140
+ return pd.read_excel(path)
141
+ raise ValueError(f"Unsupported input format {suffix!r}; expected .csv, .parquet, .xls, or .xlsx")
142
+
143
+
144
+ def _write_frame(df, path: Path) -> None:
145
+ suffix = path.suffix.lower()
146
+ path.parent.mkdir(parents=True, exist_ok=True)
147
+ if suffix == ".csv":
148
+ df.to_csv(path, index=False)
149
+ return
150
+ if suffix == ".parquet":
151
+ df.to_parquet(path, index=False)
152
+ return
153
+ if suffix in {".xls", ".xlsx"}:
154
+ df.to_excel(path, index=False)
155
+ return
156
+ raise ValueError(f"Unsupported output format {suffix!r}; expected .csv, .parquet, .xls, or .xlsx")
157
+
158
+
159
+ def _default_output_path(path: Path) -> Path:
160
+ return path.with_name(f"{path.stem}_mimic{path.suffix}")
161
+
162
+
163
+ if __name__ == "__main__": # pragma: no cover
164
+ raise SystemExit(main())
mimic/datasets.py ADDED
@@ -0,0 +1,259 @@
1
+ """Convenience loaders for the manuscript datasets."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+
7
+ import numpy as np
8
+ import pandas as pd
9
+ from sklearn.datasets import fetch_openml
10
+
11
+
12
+ @dataclass(frozen=True)
13
+ class PaperDatasetSpec:
14
+ """Pinned dataset metadata used by the manuscript experiments."""
15
+
16
+ key: str
17
+ name: str
18
+ openml_id: int | None
19
+ openml_name: str | None
20
+ openml_version: int | None
21
+ target: str
22
+ n_rows: int
23
+ minority_value: object | None
24
+ classification_columns: tuple[str, ...]
25
+ regression_columns: tuple[str, ...]
26
+ ignore_columns: tuple[str, ...] = ()
27
+
28
+
29
+ ADULT_CLASSIFICATION = (
30
+ "workclass",
31
+ "education",
32
+ "marital-status",
33
+ "occupation",
34
+ "relationship",
35
+ "race",
36
+ "sex",
37
+ "native-country",
38
+ )
39
+ ADULT_REGRESSION = ("age", "fnlwgt", "education-num", "capital-gain", "capital-loss", "hours-per-week")
40
+
41
+ BANK_CLASSIFICATION = ("job", "marital", "education", "default", "housing", "loan", "contact", "month", "poutcome")
42
+ BANK_REGRESSION = ("age", "balance", "day", "duration", "campaign", "pdays", "previous")
43
+
44
+ DEFAULT_CREDIT_CLASSIFICATION = ("SEX", "EDUCATION", "MARRIAGE", "PAY_0", "PAY_2", "PAY_3", "PAY_4", "PAY_5", "PAY_6")
45
+ DEFAULT_CREDIT_REGRESSION = (
46
+ "LIMIT_BAL",
47
+ "AGE",
48
+ "BILL_AMT1",
49
+ "BILL_AMT2",
50
+ "BILL_AMT3",
51
+ "BILL_AMT4",
52
+ "BILL_AMT5",
53
+ "BILL_AMT6",
54
+ "PAY_AMT1",
55
+ "PAY_AMT2",
56
+ "PAY_AMT3",
57
+ "PAY_AMT4",
58
+ "PAY_AMT5",
59
+ "PAY_AMT6",
60
+ )
61
+
62
+
63
+ PAPER_DATASETS: dict[str, PaperDatasetSpec] = {
64
+ "adult": PaperDatasetSpec(
65
+ key="adult",
66
+ name="Adult",
67
+ openml_id=1590,
68
+ openml_name="adult",
69
+ openml_version=2,
70
+ target="class",
71
+ n_rows=48842,
72
+ minority_value=">50K",
73
+ classification_columns=ADULT_CLASSIFICATION,
74
+ regression_columns=ADULT_REGRESSION,
75
+ ),
76
+ "bank_marketing": PaperDatasetSpec(
77
+ key="bank_marketing",
78
+ name="Bank Marketing",
79
+ openml_id=44234,
80
+ openml_name="Bank_marketing_data_set_UCI",
81
+ openml_version=None,
82
+ target="y",
83
+ n_rows=45211,
84
+ minority_value="yes",
85
+ classification_columns=BANK_CLASSIFICATION,
86
+ regression_columns=BANK_REGRESSION,
87
+ ),
88
+ "default_credit": PaperDatasetSpec(
89
+ key="default_credit",
90
+ name="Default of Credit Card Clients",
91
+ openml_id=42477,
92
+ openml_name="default-of-credit-card-clients",
93
+ openml_version=None,
94
+ target="y",
95
+ n_rows=30000,
96
+ minority_value=1,
97
+ classification_columns=DEFAULT_CREDIT_CLASSIFICATION,
98
+ regression_columns=DEFAULT_CREDIT_REGRESSION,
99
+ ignore_columns=("ID",),
100
+ ),
101
+ }
102
+
103
+ _ALIASES = {
104
+ "adult_mixed": "adult",
105
+ "bank": "bank_marketing",
106
+ "bank-marketing": "bank_marketing",
107
+ "credit": "default_credit",
108
+ "default": "default_credit",
109
+ "default-of-credit-card-clients": "default_credit",
110
+ }
111
+
112
+
113
+ def paper_dataset_names() -> list[str]:
114
+ """Return the public keys accepted by :func:`load_paper_dataset`."""
115
+
116
+ return list(PAPER_DATASETS)
117
+
118
+
119
+ def paper_dataset_registry() -> pd.DataFrame:
120
+ """Return metadata for the three pinned manuscript datasets."""
121
+
122
+ rows = []
123
+ for spec in PAPER_DATASETS.values():
124
+ rows.append(
125
+ {
126
+ "key": spec.key,
127
+ "name": spec.name,
128
+ "openml_id": spec.openml_id,
129
+ "target": spec.target,
130
+ "n_rows": spec.n_rows,
131
+ "regression_columns": len(spec.regression_columns),
132
+ "classification_columns": len(spec.classification_columns) + 1,
133
+ "ignore_columns": len(spec.ignore_columns),
134
+ }
135
+ )
136
+ return pd.DataFrame(rows)
137
+
138
+
139
+ def load_paper_dataset(
140
+ key: str,
141
+ *,
142
+ n_rows: int | None = None,
143
+ random_state: int = 0,
144
+ ) -> tuple[pd.DataFrame, dict[str, list[str]]]:
145
+ """Load one manuscript dataset and return ``(df, columns)`` for MIMIC.
146
+
147
+ The returned dataframe has a standardized binary ``label`` column with values
148
+ ``"minority"`` and ``"majority"``. The returned ``columns`` dictionary uses
149
+ MIMIC's public role names: ``ignore``, ``regression``, and
150
+ ``classification``.
151
+ """
152
+
153
+ resolved = _resolve_key(key)
154
+ spec = PAPER_DATASETS[resolved]
155
+ frame = _load_openml_frame(spec)
156
+ frame = _standardize_frame(frame, spec)
157
+ if n_rows is not None:
158
+ n_rows = int(n_rows)
159
+ if n_rows <= 0:
160
+ raise ValueError("n_rows must be positive when provided")
161
+ if n_rows < len(frame):
162
+ frame = _stratified_subsample(frame, n_rows=n_rows, random_state=random_state)
163
+ frame = frame.reset_index(drop=True)
164
+ return frame, paper_dataset_columns(frame, resolved)
165
+
166
+
167
+ def paper_dataset_columns(frame: pd.DataFrame, key: str) -> dict[str, list[str]]:
168
+ """Return MIMIC column roles for a loaded manuscript dataset."""
169
+
170
+ spec = PAPER_DATASETS[_resolve_key(key)]
171
+ ignore = [column for column in spec.ignore_columns if column in frame.columns]
172
+ regression = [column for column in spec.regression_columns if column in frame.columns and column != "label"]
173
+ classification = [
174
+ column for column in spec.classification_columns if column in frame.columns and column not in {"label", *ignore}
175
+ ]
176
+ if "label" in frame.columns:
177
+ classification.append("label")
178
+
179
+ assigned = set(ignore + regression + classification)
180
+ extra = [column for column in frame.columns if column not in assigned]
181
+ for column in extra:
182
+ if pd.api.types.is_numeric_dtype(frame[column]) and frame[column].nunique(dropna=True) > 10:
183
+ regression.append(column)
184
+ else:
185
+ classification.append(column)
186
+ return {"ignore": ignore, "regression": regression, "classification": classification}
187
+
188
+
189
+ def _resolve_key(key: str) -> str:
190
+ normalized = str(key).strip().lower().replace(" ", "_")
191
+ normalized = _ALIASES.get(normalized, normalized)
192
+ if normalized not in PAPER_DATASETS:
193
+ choices = ", ".join(sorted(PAPER_DATASETS))
194
+ raise ValueError(f"Unknown paper dataset {key!r}; expected one of {choices}")
195
+ return normalized
196
+
197
+
198
+ def _load_openml_frame(spec: PaperDatasetSpec) -> pd.DataFrame:
199
+ try:
200
+ if spec.openml_id is not None:
201
+ dataset = fetch_openml(data_id=spec.openml_id, as_frame=True)
202
+ else:
203
+ dataset = fetch_openml(name=spec.openml_name, version=spec.openml_version, as_frame=True)
204
+ except Exception:
205
+ if spec.openml_name is None:
206
+ raise
207
+ dataset = fetch_openml(name=spec.openml_name, version=spec.openml_version, as_frame=True)
208
+ if getattr(dataset, "frame", None) is not None:
209
+ return dataset.frame.copy()
210
+ frame = pd.DataFrame(dataset.data).copy()
211
+ frame[spec.target] = dataset.target
212
+ return frame
213
+
214
+
215
+ def _standardize_frame(frame: pd.DataFrame, spec: PaperDatasetSpec) -> pd.DataFrame:
216
+ frame = frame.copy().replace("?", np.nan)
217
+ target = _resolve_target(frame, spec.target)
218
+ labels = frame[target]
219
+ features = frame.drop(columns=[target])
220
+ if spec.minority_value is None:
221
+ counts = labels.value_counts(dropna=True)
222
+ if len(counts) != 2:
223
+ raise ValueError(f"Expected binary target for {spec.key}, found {len(counts)} classes")
224
+ minority_value = counts.idxmin()
225
+ else:
226
+ minority_value = spec.minority_value
227
+ normalized = labels.astype(str).eq(str(minority_value))
228
+ features["label"] = np.where(normalized, "minority", "majority")
229
+ return features
230
+
231
+
232
+ def _resolve_target(frame: pd.DataFrame, target: str) -> str:
233
+ if target in frame.columns:
234
+ return target
235
+ lower = {str(column).lower(): column for column in frame.columns}
236
+ if target.lower() in lower:
237
+ return lower[target.lower()]
238
+ candidates = [column for column in frame.columns if str(column).lower() in {"class", "target", "y"}]
239
+ if candidates:
240
+ return candidates[-1]
241
+ raise ValueError(f"Target column {target!r} not found")
242
+
243
+
244
+ def _stratified_subsample(frame: pd.DataFrame, *, n_rows: int, random_state: int) -> pd.DataFrame:
245
+ if "label" not in frame.columns:
246
+ return frame.sample(n=n_rows, random_state=random_state).reset_index(drop=True)
247
+ fractions = frame["label"].value_counts(normalize=True)
248
+ pieces = []
249
+ remaining = n_rows
250
+ labels = list(fractions.index)
251
+ for index, label in enumerate(labels):
252
+ group = frame.loc[frame["label"].eq(label)]
253
+ if index == len(labels) - 1:
254
+ size = min(len(group), remaining)
255
+ else:
256
+ size = min(len(group), max(1, int(round(n_rows * float(fractions[label])))))
257
+ remaining -= size
258
+ pieces.append(group.sample(n=size, random_state=random_state))
259
+ return pd.concat(pieces).sample(frac=1.0, random_state=random_state).reset_index(drop=True)