mimic-tabular 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mimic/__init__.py +34 -0
- mimic/_torch_utils.py +23 -0
- mimic/_version.py +3 -0
- mimic/cli.py +164 -0
- mimic/datasets.py +259 -0
- mimic/decoders.py +912 -0
- mimic/diagnostics.py +304 -0
- mimic/encoders.py +685 -0
- mimic/iterated.py +373 -0
- mimic/mimic.py +2156 -0
- mimic/policies.py +28 -0
- mimic_experiments/__init__.py +2 -0
- mimic_experiments/q1_smote_roc.py +1016 -0
- mimic_experiments/q2_adasyn.py +482 -0
- mimic_experiments/q3_geometric_smote.py +669 -0
- mimic_tabular-0.1.3.dist-info/METADATA +416 -0
- mimic_tabular-0.1.3.dist-info/RECORD +26 -0
- mimic_tabular-0.1.3.dist-info/WHEEL +5 -0
- mimic_tabular-0.1.3.dist-info/entry_points.txt +2 -0
- mimic_tabular-0.1.3.dist-info/licenses/LICENSE +21 -0
- mimic_tabular-0.1.3.dist-info/top_level.txt +3 -0
- mimic_vision/__init__.py +53 -0
- mimic_vision/datasets.py +395 -0
- mimic_vision/groups.py +34 -0
- mimic_vision/synthesis.py +203 -0
- mimic_vision/visualization.py +359 -0
mimic/__init__.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""MIMIC public API."""
|
|
2
|
+
|
|
3
|
+
from ._version import __version__
|
|
4
|
+
from .decoders import ForestConditionalSampler, IdentityDecoder, LinearMixedFeatureDecoder, MixedFeatureDecoder, NeuralConditionalSampler
|
|
5
|
+
from .datasets import load_paper_dataset, paper_dataset_columns, paper_dataset_names, paper_dataset_registry
|
|
6
|
+
from .encoders import IdentityEncoder, RandomForestPathEncoder, ResNetEncoder, SharedFeatureGroup, SharedResNetEncoder
|
|
7
|
+
from .iterated import IteratedMIMIC
|
|
8
|
+
from .mimic import MIMIC, NearestNeighborPrivacyFilter, mimic_data, sample, sample_dataframe
|
|
9
|
+
from .policies import GenerationPolicy
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"__version__",
|
|
13
|
+
"GenerationPolicy",
|
|
14
|
+
"ForestConditionalSampler",
|
|
15
|
+
"IdentityDecoder",
|
|
16
|
+
"IdentityEncoder",
|
|
17
|
+
"IteratedMIMIC",
|
|
18
|
+
"LinearMixedFeatureDecoder",
|
|
19
|
+
"MIMIC",
|
|
20
|
+
"MixedFeatureDecoder",
|
|
21
|
+
"NeuralConditionalSampler",
|
|
22
|
+
"NearestNeighborPrivacyFilter",
|
|
23
|
+
"load_paper_dataset",
|
|
24
|
+
"paper_dataset_columns",
|
|
25
|
+
"paper_dataset_names",
|
|
26
|
+
"paper_dataset_registry",
|
|
27
|
+
"RandomForestPathEncoder",
|
|
28
|
+
"ResNetEncoder",
|
|
29
|
+
"SharedFeatureGroup",
|
|
30
|
+
"SharedResNetEncoder",
|
|
31
|
+
"mimic_data",
|
|
32
|
+
"sample",
|
|
33
|
+
"sample_dataframe",
|
|
34
|
+
]
|
mimic/_torch_utils.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Utilities for keeping optional PyTorch state local to model fitting."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from functools import wraps
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
import torch
|
|
9
|
+
except Exception: # pragma: no cover - import availability is environment dependent
|
|
10
|
+
torch = None
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def preserve_torch_rng(method):
|
|
14
|
+
"""Restore PyTorch's process-wide RNG state after seeded model fitting."""
|
|
15
|
+
|
|
16
|
+
@wraps(method)
|
|
17
|
+
def wrapped(self, *args, **kwargs):
|
|
18
|
+
if torch is None or getattr(self, "random_state", None) is None:
|
|
19
|
+
return method(self, *args, **kwargs)
|
|
20
|
+
with torch.random.fork_rng(devices=None):
|
|
21
|
+
return method(self, *args, **kwargs)
|
|
22
|
+
|
|
23
|
+
return wrapped
|
mimic/_version.py
ADDED
mimic/cli.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""Command-line interface for one-shot MIMIC data generation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from ._version import __version__
|
|
10
|
+
from .mimic import mimic_data
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def main(argv: list[str] | None = None) -> int:
|
|
14
|
+
parser = _build_parser()
|
|
15
|
+
args = parser.parse_args(argv)
|
|
16
|
+
|
|
17
|
+
input_path = Path(args.filename)
|
|
18
|
+
if not input_path.exists():
|
|
19
|
+
parser.error(f"input file does not exist: {input_path}")
|
|
20
|
+
|
|
21
|
+
columns = _parse_columns(args.columns, parser)
|
|
22
|
+
try:
|
|
23
|
+
df = _read_frame(input_path)
|
|
24
|
+
except ValueError as exc:
|
|
25
|
+
parser.error(str(exc))
|
|
26
|
+
try:
|
|
27
|
+
synthetic = mimic_data(
|
|
28
|
+
df,
|
|
29
|
+
columns=columns,
|
|
30
|
+
mode=args.mode,
|
|
31
|
+
capacity=args.capacity,
|
|
32
|
+
save_model=args.save_model,
|
|
33
|
+
load_model=args.load_model,
|
|
34
|
+
refit=args.refit,
|
|
35
|
+
random_state=args.random_state,
|
|
36
|
+
)
|
|
37
|
+
except (OSError, ValueError) as exc:
|
|
38
|
+
parser.error(str(exc))
|
|
39
|
+
|
|
40
|
+
output_path = Path(args.output) if args.output is not None else _default_output_path(input_path)
|
|
41
|
+
try:
|
|
42
|
+
_write_frame(synthetic, output_path)
|
|
43
|
+
except ValueError as exc:
|
|
44
|
+
parser.error(str(exc))
|
|
45
|
+
print(f"Wrote {len(synthetic)} rows to {output_path}")
|
|
46
|
+
return 0
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
50
|
+
parser = argparse.ArgumentParser(
|
|
51
|
+
prog="mimic-data",
|
|
52
|
+
description=(
|
|
53
|
+
"Fit MIMIC on a tabular data file and write a synthetic dataset with "
|
|
54
|
+
"the same number of rows. CSV, Parquet, and Excel files are supported."
|
|
55
|
+
),
|
|
56
|
+
)
|
|
57
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
58
|
+
parser.add_argument("filename", help="Input .csv, .parquet, .xls, or .xlsx file.")
|
|
59
|
+
parser.add_argument(
|
|
60
|
+
"-o",
|
|
61
|
+
"--output",
|
|
62
|
+
help="Output filename. Defaults to '<input_stem>_mimic<input_suffix>' in the same directory.",
|
|
63
|
+
)
|
|
64
|
+
parser.add_argument(
|
|
65
|
+
"--columns",
|
|
66
|
+
default="auto",
|
|
67
|
+
help=(
|
|
68
|
+
"Column role specification. Use 'auto' for heuristic inference, or pass a JSON mapping "
|
|
69
|
+
"such as '{\"ignore\":[\"id\"],\"regression\":[\"age\"],\"classification\":[\"segment\"]}'. "
|
|
70
|
+
"Prefix with @ to read the JSON mapping from a file."
|
|
71
|
+
),
|
|
72
|
+
)
|
|
73
|
+
parser.add_argument(
|
|
74
|
+
"--mode",
|
|
75
|
+
default="factorised",
|
|
76
|
+
choices=["identity", "direct", "factorised", "joint", "0", "1", "2", "3"],
|
|
77
|
+
help="MIMIC preset mode. Defaults to %(default)s.",
|
|
78
|
+
)
|
|
79
|
+
parser.add_argument(
|
|
80
|
+
"--capacity",
|
|
81
|
+
type=float,
|
|
82
|
+
default=0.25,
|
|
83
|
+
help="Model capacity in [0, 1]. Defaults to %(default)s.",
|
|
84
|
+
)
|
|
85
|
+
parser.add_argument(
|
|
86
|
+
"--random-state",
|
|
87
|
+
type=int,
|
|
88
|
+
default=None,
|
|
89
|
+
help="Optional random seed for reproducible generation.",
|
|
90
|
+
)
|
|
91
|
+
parser.add_argument(
|
|
92
|
+
"--save-model",
|
|
93
|
+
default=None,
|
|
94
|
+
help="Optional path where the fitted MIMIC model should be saved.",
|
|
95
|
+
)
|
|
96
|
+
parser.add_argument(
|
|
97
|
+
"--load-model",
|
|
98
|
+
default=None,
|
|
99
|
+
help="Optional path to a fitted MIMIC model to load before sampling.",
|
|
100
|
+
)
|
|
101
|
+
parser.add_argument(
|
|
102
|
+
"--refit",
|
|
103
|
+
action="store_true",
|
|
104
|
+
help="Fit a fresh model even when --load-model is supplied.",
|
|
105
|
+
)
|
|
106
|
+
return parser
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _parse_columns(value: str, parser: argparse.ArgumentParser):
|
|
110
|
+
if value in {"auto", "None", "none"}:
|
|
111
|
+
return "auto" if value == "auto" else None
|
|
112
|
+
try:
|
|
113
|
+
if value.startswith("@"):
|
|
114
|
+
text = Path(value[1:]).read_text()
|
|
115
|
+
else:
|
|
116
|
+
text = value
|
|
117
|
+
parsed = json.loads(text)
|
|
118
|
+
except OSError as exc:
|
|
119
|
+
parser.error(f"could not read columns file {value!r}: {exc}")
|
|
120
|
+
except json.JSONDecodeError as exc:
|
|
121
|
+
parser.error(f"--columns must be 'auto' or valid JSON: {exc}")
|
|
122
|
+
if not isinstance(parsed, dict):
|
|
123
|
+
parser.error("--columns JSON must be an object mapping roles to column lists")
|
|
124
|
+
return parsed
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _read_frame(path: Path):
|
|
128
|
+
suffix = path.suffix.lower()
|
|
129
|
+
if suffix == ".csv":
|
|
130
|
+
import pandas as pd
|
|
131
|
+
|
|
132
|
+
return pd.read_csv(path)
|
|
133
|
+
if suffix == ".parquet":
|
|
134
|
+
import pandas as pd
|
|
135
|
+
|
|
136
|
+
return pd.read_parquet(path)
|
|
137
|
+
if suffix in {".xls", ".xlsx"}:
|
|
138
|
+
import pandas as pd
|
|
139
|
+
|
|
140
|
+
return pd.read_excel(path)
|
|
141
|
+
raise ValueError(f"Unsupported input format {suffix!r}; expected .csv, .parquet, .xls, or .xlsx")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _write_frame(df, path: Path) -> None:
|
|
145
|
+
suffix = path.suffix.lower()
|
|
146
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
147
|
+
if suffix == ".csv":
|
|
148
|
+
df.to_csv(path, index=False)
|
|
149
|
+
return
|
|
150
|
+
if suffix == ".parquet":
|
|
151
|
+
df.to_parquet(path, index=False)
|
|
152
|
+
return
|
|
153
|
+
if suffix in {".xls", ".xlsx"}:
|
|
154
|
+
df.to_excel(path, index=False)
|
|
155
|
+
return
|
|
156
|
+
raise ValueError(f"Unsupported output format {suffix!r}; expected .csv, .parquet, .xls, or .xlsx")
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _default_output_path(path: Path) -> Path:
|
|
160
|
+
return path.with_name(f"{path.stem}_mimic{path.suffix}")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
if __name__ == "__main__": # pragma: no cover
|
|
164
|
+
raise SystemExit(main())
|
mimic/datasets.py
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"""Convenience loaders for the manuscript datasets."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import pandas as pd
|
|
9
|
+
from sklearn.datasets import fetch_openml
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class PaperDatasetSpec:
|
|
14
|
+
"""Pinned dataset metadata used by the manuscript experiments."""
|
|
15
|
+
|
|
16
|
+
key: str
|
|
17
|
+
name: str
|
|
18
|
+
openml_id: int | None
|
|
19
|
+
openml_name: str | None
|
|
20
|
+
openml_version: int | None
|
|
21
|
+
target: str
|
|
22
|
+
n_rows: int
|
|
23
|
+
minority_value: object | None
|
|
24
|
+
classification_columns: tuple[str, ...]
|
|
25
|
+
regression_columns: tuple[str, ...]
|
|
26
|
+
ignore_columns: tuple[str, ...] = ()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
ADULT_CLASSIFICATION = (
|
|
30
|
+
"workclass",
|
|
31
|
+
"education",
|
|
32
|
+
"marital-status",
|
|
33
|
+
"occupation",
|
|
34
|
+
"relationship",
|
|
35
|
+
"race",
|
|
36
|
+
"sex",
|
|
37
|
+
"native-country",
|
|
38
|
+
)
|
|
39
|
+
ADULT_REGRESSION = ("age", "fnlwgt", "education-num", "capital-gain", "capital-loss", "hours-per-week")
|
|
40
|
+
|
|
41
|
+
BANK_CLASSIFICATION = ("job", "marital", "education", "default", "housing", "loan", "contact", "month", "poutcome")
|
|
42
|
+
BANK_REGRESSION = ("age", "balance", "day", "duration", "campaign", "pdays", "previous")
|
|
43
|
+
|
|
44
|
+
DEFAULT_CREDIT_CLASSIFICATION = ("SEX", "EDUCATION", "MARRIAGE", "PAY_0", "PAY_2", "PAY_3", "PAY_4", "PAY_5", "PAY_6")
|
|
45
|
+
DEFAULT_CREDIT_REGRESSION = (
|
|
46
|
+
"LIMIT_BAL",
|
|
47
|
+
"AGE",
|
|
48
|
+
"BILL_AMT1",
|
|
49
|
+
"BILL_AMT2",
|
|
50
|
+
"BILL_AMT3",
|
|
51
|
+
"BILL_AMT4",
|
|
52
|
+
"BILL_AMT5",
|
|
53
|
+
"BILL_AMT6",
|
|
54
|
+
"PAY_AMT1",
|
|
55
|
+
"PAY_AMT2",
|
|
56
|
+
"PAY_AMT3",
|
|
57
|
+
"PAY_AMT4",
|
|
58
|
+
"PAY_AMT5",
|
|
59
|
+
"PAY_AMT6",
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
PAPER_DATASETS: dict[str, PaperDatasetSpec] = {
|
|
64
|
+
"adult": PaperDatasetSpec(
|
|
65
|
+
key="adult",
|
|
66
|
+
name="Adult",
|
|
67
|
+
openml_id=1590,
|
|
68
|
+
openml_name="adult",
|
|
69
|
+
openml_version=2,
|
|
70
|
+
target="class",
|
|
71
|
+
n_rows=48842,
|
|
72
|
+
minority_value=">50K",
|
|
73
|
+
classification_columns=ADULT_CLASSIFICATION,
|
|
74
|
+
regression_columns=ADULT_REGRESSION,
|
|
75
|
+
),
|
|
76
|
+
"bank_marketing": PaperDatasetSpec(
|
|
77
|
+
key="bank_marketing",
|
|
78
|
+
name="Bank Marketing",
|
|
79
|
+
openml_id=44234,
|
|
80
|
+
openml_name="Bank_marketing_data_set_UCI",
|
|
81
|
+
openml_version=None,
|
|
82
|
+
target="y",
|
|
83
|
+
n_rows=45211,
|
|
84
|
+
minority_value="yes",
|
|
85
|
+
classification_columns=BANK_CLASSIFICATION,
|
|
86
|
+
regression_columns=BANK_REGRESSION,
|
|
87
|
+
),
|
|
88
|
+
"default_credit": PaperDatasetSpec(
|
|
89
|
+
key="default_credit",
|
|
90
|
+
name="Default of Credit Card Clients",
|
|
91
|
+
openml_id=42477,
|
|
92
|
+
openml_name="default-of-credit-card-clients",
|
|
93
|
+
openml_version=None,
|
|
94
|
+
target="y",
|
|
95
|
+
n_rows=30000,
|
|
96
|
+
minority_value=1,
|
|
97
|
+
classification_columns=DEFAULT_CREDIT_CLASSIFICATION,
|
|
98
|
+
regression_columns=DEFAULT_CREDIT_REGRESSION,
|
|
99
|
+
ignore_columns=("ID",),
|
|
100
|
+
),
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
_ALIASES = {
|
|
104
|
+
"adult_mixed": "adult",
|
|
105
|
+
"bank": "bank_marketing",
|
|
106
|
+
"bank-marketing": "bank_marketing",
|
|
107
|
+
"credit": "default_credit",
|
|
108
|
+
"default": "default_credit",
|
|
109
|
+
"default-of-credit-card-clients": "default_credit",
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def paper_dataset_names() -> list[str]:
|
|
114
|
+
"""Return the public keys accepted by :func:`load_paper_dataset`."""
|
|
115
|
+
|
|
116
|
+
return list(PAPER_DATASETS)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def paper_dataset_registry() -> pd.DataFrame:
|
|
120
|
+
"""Return metadata for the three pinned manuscript datasets."""
|
|
121
|
+
|
|
122
|
+
rows = []
|
|
123
|
+
for spec in PAPER_DATASETS.values():
|
|
124
|
+
rows.append(
|
|
125
|
+
{
|
|
126
|
+
"key": spec.key,
|
|
127
|
+
"name": spec.name,
|
|
128
|
+
"openml_id": spec.openml_id,
|
|
129
|
+
"target": spec.target,
|
|
130
|
+
"n_rows": spec.n_rows,
|
|
131
|
+
"regression_columns": len(spec.regression_columns),
|
|
132
|
+
"classification_columns": len(spec.classification_columns) + 1,
|
|
133
|
+
"ignore_columns": len(spec.ignore_columns),
|
|
134
|
+
}
|
|
135
|
+
)
|
|
136
|
+
return pd.DataFrame(rows)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def load_paper_dataset(
|
|
140
|
+
key: str,
|
|
141
|
+
*,
|
|
142
|
+
n_rows: int | None = None,
|
|
143
|
+
random_state: int = 0,
|
|
144
|
+
) -> tuple[pd.DataFrame, dict[str, list[str]]]:
|
|
145
|
+
"""Load one manuscript dataset and return ``(df, columns)`` for MIMIC.
|
|
146
|
+
|
|
147
|
+
The returned dataframe has a standardized binary ``label`` column with values
|
|
148
|
+
``"minority"`` and ``"majority"``. The returned ``columns`` dictionary uses
|
|
149
|
+
MIMIC's public role names: ``ignore``, ``regression``, and
|
|
150
|
+
``classification``.
|
|
151
|
+
"""
|
|
152
|
+
|
|
153
|
+
resolved = _resolve_key(key)
|
|
154
|
+
spec = PAPER_DATASETS[resolved]
|
|
155
|
+
frame = _load_openml_frame(spec)
|
|
156
|
+
frame = _standardize_frame(frame, spec)
|
|
157
|
+
if n_rows is not None:
|
|
158
|
+
n_rows = int(n_rows)
|
|
159
|
+
if n_rows <= 0:
|
|
160
|
+
raise ValueError("n_rows must be positive when provided")
|
|
161
|
+
if n_rows < len(frame):
|
|
162
|
+
frame = _stratified_subsample(frame, n_rows=n_rows, random_state=random_state)
|
|
163
|
+
frame = frame.reset_index(drop=True)
|
|
164
|
+
return frame, paper_dataset_columns(frame, resolved)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def paper_dataset_columns(frame: pd.DataFrame, key: str) -> dict[str, list[str]]:
|
|
168
|
+
"""Return MIMIC column roles for a loaded manuscript dataset."""
|
|
169
|
+
|
|
170
|
+
spec = PAPER_DATASETS[_resolve_key(key)]
|
|
171
|
+
ignore = [column for column in spec.ignore_columns if column in frame.columns]
|
|
172
|
+
regression = [column for column in spec.regression_columns if column in frame.columns and column != "label"]
|
|
173
|
+
classification = [
|
|
174
|
+
column for column in spec.classification_columns if column in frame.columns and column not in {"label", *ignore}
|
|
175
|
+
]
|
|
176
|
+
if "label" in frame.columns:
|
|
177
|
+
classification.append("label")
|
|
178
|
+
|
|
179
|
+
assigned = set(ignore + regression + classification)
|
|
180
|
+
extra = [column for column in frame.columns if column not in assigned]
|
|
181
|
+
for column in extra:
|
|
182
|
+
if pd.api.types.is_numeric_dtype(frame[column]) and frame[column].nunique(dropna=True) > 10:
|
|
183
|
+
regression.append(column)
|
|
184
|
+
else:
|
|
185
|
+
classification.append(column)
|
|
186
|
+
return {"ignore": ignore, "regression": regression, "classification": classification}
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _resolve_key(key: str) -> str:
|
|
190
|
+
normalized = str(key).strip().lower().replace(" ", "_")
|
|
191
|
+
normalized = _ALIASES.get(normalized, normalized)
|
|
192
|
+
if normalized not in PAPER_DATASETS:
|
|
193
|
+
choices = ", ".join(sorted(PAPER_DATASETS))
|
|
194
|
+
raise ValueError(f"Unknown paper dataset {key!r}; expected one of {choices}")
|
|
195
|
+
return normalized
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _load_openml_frame(spec: PaperDatasetSpec) -> pd.DataFrame:
|
|
199
|
+
try:
|
|
200
|
+
if spec.openml_id is not None:
|
|
201
|
+
dataset = fetch_openml(data_id=spec.openml_id, as_frame=True)
|
|
202
|
+
else:
|
|
203
|
+
dataset = fetch_openml(name=spec.openml_name, version=spec.openml_version, as_frame=True)
|
|
204
|
+
except Exception:
|
|
205
|
+
if spec.openml_name is None:
|
|
206
|
+
raise
|
|
207
|
+
dataset = fetch_openml(name=spec.openml_name, version=spec.openml_version, as_frame=True)
|
|
208
|
+
if getattr(dataset, "frame", None) is not None:
|
|
209
|
+
return dataset.frame.copy()
|
|
210
|
+
frame = pd.DataFrame(dataset.data).copy()
|
|
211
|
+
frame[spec.target] = dataset.target
|
|
212
|
+
return frame
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _standardize_frame(frame: pd.DataFrame, spec: PaperDatasetSpec) -> pd.DataFrame:
|
|
216
|
+
frame = frame.copy().replace("?", np.nan)
|
|
217
|
+
target = _resolve_target(frame, spec.target)
|
|
218
|
+
labels = frame[target]
|
|
219
|
+
features = frame.drop(columns=[target])
|
|
220
|
+
if spec.minority_value is None:
|
|
221
|
+
counts = labels.value_counts(dropna=True)
|
|
222
|
+
if len(counts) != 2:
|
|
223
|
+
raise ValueError(f"Expected binary target for {spec.key}, found {len(counts)} classes")
|
|
224
|
+
minority_value = counts.idxmin()
|
|
225
|
+
else:
|
|
226
|
+
minority_value = spec.minority_value
|
|
227
|
+
normalized = labels.astype(str).eq(str(minority_value))
|
|
228
|
+
features["label"] = np.where(normalized, "minority", "majority")
|
|
229
|
+
return features
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _resolve_target(frame: pd.DataFrame, target: str) -> str:
|
|
233
|
+
if target in frame.columns:
|
|
234
|
+
return target
|
|
235
|
+
lower = {str(column).lower(): column for column in frame.columns}
|
|
236
|
+
if target.lower() in lower:
|
|
237
|
+
return lower[target.lower()]
|
|
238
|
+
candidates = [column for column in frame.columns if str(column).lower() in {"class", "target", "y"}]
|
|
239
|
+
if candidates:
|
|
240
|
+
return candidates[-1]
|
|
241
|
+
raise ValueError(f"Target column {target!r} not found")
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _stratified_subsample(frame: pd.DataFrame, *, n_rows: int, random_state: int) -> pd.DataFrame:
|
|
245
|
+
if "label" not in frame.columns:
|
|
246
|
+
return frame.sample(n=n_rows, random_state=random_state).reset_index(drop=True)
|
|
247
|
+
fractions = frame["label"].value_counts(normalize=True)
|
|
248
|
+
pieces = []
|
|
249
|
+
remaining = n_rows
|
|
250
|
+
labels = list(fractions.index)
|
|
251
|
+
for index, label in enumerate(labels):
|
|
252
|
+
group = frame.loc[frame["label"].eq(label)]
|
|
253
|
+
if index == len(labels) - 1:
|
|
254
|
+
size = min(len(group), remaining)
|
|
255
|
+
else:
|
|
256
|
+
size = min(len(group), max(1, int(round(n_rows * float(fractions[label])))))
|
|
257
|
+
remaining -= size
|
|
258
|
+
pieces.append(group.sample(n=size, random_state=random_state))
|
|
259
|
+
return pd.concat(pieces).sample(frac=1.0, random_state=random_state).reset_index(drop=True)
|