synthyverse 0.2.8__tar.gz → 0.2.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {synthyverse-0.2.8 → synthyverse-0.2.9}/PKG-INFO +4 -2
- {synthyverse-0.2.8 → synthyverse-0.2.9}/pyproject.toml +2 -1
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/evaluation/fidelity.py +89 -6
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/evaluation/ml.py +86 -26
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/evaluation/privacy.py +4 -1
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/base.py +27 -6
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/cdtd_generator/cdtd.py +3 -2
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/config.py +44 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/ctgan_generator/ct_gan.py +8 -1
- synthyverse-0.2.9/synthyverse/generators/ctgan_generator/data_transformer.py +284 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/ctgan_generator/synthesizer.py +6 -3
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/dgm_utils.py +2 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/forestdiffusion_generator/fd.py +11 -5
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/forestdiffusion_generator/model.py +1 -1
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabargn_generator/tabargn.py +5 -1
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabbyflow_generator/flow_model.py +1 -2
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabbyflow_generator/tabbyflow.py +9 -7
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabcascade_generator/highres.py +1 -1
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabcascade_generator/lowres.py +1 -1
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabcascade_generator/tabcascade.py +8 -8
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabddpm_generator/gaussian_multinomial_diffsuion.py +6 -2
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabddpm_generator/tabddpm.py +53 -14
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabdiff_generator/tabdiff.py +3 -2
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabsyn_generator/tabsyn.py +32 -31
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tvae_generator/synthesizer.py +6 -3
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tvae_generator/tvae.py +8 -1
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/univariate_generator/univariate.py +22 -7
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse.egg-info/PKG-INFO +4 -2
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse.egg-info/SOURCES.txt +1 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse.egg-info/requires.txt +1 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/LICENSE +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/LICENSES/Apache-2.0.txt +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/LICENSES/CTGAN-BSL-1.1.txt +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/LICENSES/MIT.txt +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/MANIFEST.in +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/NOTICE +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/README.md +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/THIRD_PARTY_NOTICES.md +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/logo/logo.png +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/setup.cfg +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/benchmark/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/benchmark/synthesis.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/evaluation/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/evaluation/base.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/evaluation/distances.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/evaluation/eval.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/evaluation/preprocessing.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/evaluation/utility.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/_optional.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/arf_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/arf_generator/arf.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/arf_generator/model.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/arf_generator/utils.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/cdtd_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/cdtd_generator/layers.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/cdtd_generator/utils.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/ctgan_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/forestdiffusion_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/forestdiffusion_generator/utils/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/forestdiffusion_generator/utils/diffusion.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/forestdiffusion_generator/utils/utils_diffusion.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/forestdiffusion_generator/utils/writer.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/persistence.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/smote_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/smote_generator/smote.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/synthpop_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/synthpop_generator/synthpop.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabargn_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabargn_generator/encoding.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabargn_generator/model.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabbyflow_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabcascade_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabcascade_generator/disttree.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabcascade_generator/encoder.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabddpm_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabddpm_generator/ddpm_utils.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabddpm_generator/modules.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabdiff_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabdiff_generator/diffusion.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabdiff_generator/modules.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabdiff_generator/noise_schedule.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabdiff_generator/transformer.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabpfn_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabpfn_generator/tabpfn.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabpfn_generator/tabpfn_extensions/unsupervised.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabpfn_generator/tabpfn_extensions/utils.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabsyn_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabsyn_generator/diffusion.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tabsyn_generator/vae.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/tvae_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/univariate_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/utrees_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/utrees_generator/_utrees/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/utrees_generator/_utrees/baltobot.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/utrees_generator/_utrees/kdi_quantizer.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/utrees_generator/_utrees/unmasking_trees.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/utrees_generator/kditransform/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/utrees_generator/kditransform/kdi_discretizer.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/utrees_generator/kditransform/kdi_transformer.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/utrees_generator/kditransform/ksum.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/utrees_generator/utrees.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/xgbddpm_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/xgbddpm_generator/xgbddpm.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/xgbdiffusion_generator/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/generators/xgbdiffusion_generator/xgbdiffusion.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/utils/__init__.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/utils/reproducibility.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse/utils/utils.py +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse.egg-info/dependency_links.txt +0 -0
- {synthyverse-0.2.8 → synthyverse-0.2.9}/synthyverse.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: synthyverse
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.9
|
|
4
4
|
Summary: Synthetic data generation and evaluation library
|
|
5
5
|
Author: Jim Achterberg, Saif Ul Islam, Zia Ur Rehman
|
|
6
6
|
License: MIT License
|
|
@@ -42,6 +42,7 @@ Requires-Dist: einops
|
|
|
42
42
|
Requires-Dist: imbalanced-learn
|
|
43
43
|
Requires-Dist: xgb-diffusion
|
|
44
44
|
Requires-Dist: torchdiffeq>=0.2.5
|
|
45
|
+
Requires-Dist: structured-data-models>=0.2.1
|
|
45
46
|
Provides-Extra: ctgan
|
|
46
47
|
Requires-Dist: ctgan==0.12.0; extra == "ctgan"
|
|
47
48
|
Provides-Extra: tabpfn
|
|
@@ -54,6 +55,7 @@ Requires-Dist: sphinx-copybutton; extra == "docs"
|
|
|
54
55
|
Requires-Dist: sphinx-autodoc-typehints; extra == "docs"
|
|
55
56
|
Requires-Dist: sphinxcontrib-mermaid; extra == "docs"
|
|
56
57
|
Requires-Dist: linkify-it-py; extra == "docs"
|
|
58
|
+
Dynamic: license-file
|
|
57
59
|
|
|
58
60
|
<table align="center" border="0">
|
|
59
61
|
<tr>
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "synthyverse"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.9"
|
|
8
8
|
description = "Synthetic data generation and evaluation library"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.12"
|
|
@@ -45,6 +45,7 @@ dependencies = [
|
|
|
45
45
|
"imbalanced-learn",
|
|
46
46
|
"xgb-diffusion",
|
|
47
47
|
"torchdiffeq>=0.2.5",
|
|
48
|
+
"structured-data-models>=0.2.1",
|
|
48
49
|
]
|
|
49
50
|
|
|
50
51
|
[tool.setuptools]
|
|
@@ -17,6 +17,7 @@ matplotlib.use("Agg")
|
|
|
17
17
|
import matplotlib.pyplot as plt
|
|
18
18
|
import seaborn as sns
|
|
19
19
|
from sklearn.metrics import mutual_info_score
|
|
20
|
+
from sklearn.model_selection import train_test_split
|
|
20
21
|
|
|
21
22
|
from scipy.stats import (
|
|
22
23
|
spearmanr,
|
|
@@ -25,7 +26,7 @@ from scipy.stats import (
|
|
|
25
26
|
)
|
|
26
27
|
|
|
27
28
|
from .base import BaseMetric
|
|
28
|
-
from .ml import cv_ml_task, resolve_model_name
|
|
29
|
+
from .ml import cv_ml_task, ml_task, resolve_model_name
|
|
29
30
|
from .distances import (
|
|
30
31
|
FastGowerNN,
|
|
31
32
|
js_robust,
|
|
@@ -56,7 +57,7 @@ def _metric_key_part(value) -> str:
|
|
|
56
57
|
class ClassifierTwoSampleTest(BaseMetric):
|
|
57
58
|
"""ROCAUC score of a classifier that distinguishes synthetic from real data.
|
|
58
59
|
|
|
59
|
-
|
|
60
|
+
Scores closer to 0.5 indicate better quality synthetic data (harder to distinguish from real).
|
|
60
61
|
|
|
61
62
|
Args:
|
|
62
63
|
random_state (int): Random seed for reproducibility. Default: 0.
|
|
@@ -68,7 +69,15 @@ class ClassifierTwoSampleTest(BaseMetric):
|
|
|
68
69
|
and other models use no extra parameters.
|
|
69
70
|
nfold (int): Number of cross-validation folds. Default: 3.
|
|
70
71
|
test_repeats (int): Number of synthetic subsamples to score against X_test
|
|
71
|
-
when X_test is provided.
|
|
72
|
+
when X_test is provided. Each repeat uses a different sampling and
|
|
73
|
+
evaluation seed. Default: 3.
|
|
74
|
+
scoring (str): "cv" for cross-validation or "holdout" for an
|
|
75
|
+
80:20 train/test split. With XGBoost early stopping, 1/8 of the
|
|
76
|
+
training split is reserved for validation. "cv_holdout" uses CV on
|
|
77
|
+
the 80% training split to select XGBoost boosting rounds, then
|
|
78
|
+
retrains on that split and scores on the 20% test split. Falls back
|
|
79
|
+
to "holdout" with a warning without XGBoost early stopping.
|
|
80
|
+
Default: "cv".
|
|
72
81
|
|
|
73
82
|
Outputs:
|
|
74
83
|
"c2st.auc" and, when X_test is provided, "c2st.test.auc";
|
|
@@ -102,8 +111,12 @@ class ClassifierTwoSampleTest(BaseMetric):
|
|
|
102
111
|
nfold: int = 3,
|
|
103
112
|
random_state: int = 0,
|
|
104
113
|
test_repeats: int = 3,
|
|
114
|
+
scoring: str = "cv",
|
|
105
115
|
):
|
|
106
116
|
super().__init__(random_state=random_state)
|
|
117
|
+
if scoring not in {"cv", "holdout", "cv_holdout"}:
|
|
118
|
+
raise ValueError('scoring must be "cv", "holdout", or "cv_holdout".')
|
|
119
|
+
self.scoring = scoring
|
|
107
120
|
self.nfold = int(nfold)
|
|
108
121
|
if self.nfold < 2:
|
|
109
122
|
raise ValueError("nfold must be >= 2.")
|
|
@@ -119,6 +132,17 @@ class ClassifierTwoSampleTest(BaseMetric):
|
|
|
119
132
|
)
|
|
120
133
|
else:
|
|
121
134
|
self.model_params = model_params.copy()
|
|
135
|
+
if self.scoring == "cv_holdout" and (
|
|
136
|
+
resolve_model_name(self.model_name) != "xgboost"
|
|
137
|
+
or self.model_params.get("early_stopping_rounds") is None
|
|
138
|
+
):
|
|
139
|
+
warnings.warn(
|
|
140
|
+
'scoring="cv_holdout" requires XGBoost with early_stopping_rounds; '
|
|
141
|
+
'falling back to "holdout".',
|
|
142
|
+
UserWarning,
|
|
143
|
+
stacklevel=2,
|
|
144
|
+
)
|
|
145
|
+
self.scoring = "holdout"
|
|
122
146
|
|
|
123
147
|
def _evaluate(
|
|
124
148
|
self,
|
|
@@ -128,7 +152,9 @@ class ClassifierTwoSampleTest(BaseMetric):
|
|
|
128
152
|
discrete_features: list = None,
|
|
129
153
|
):
|
|
130
154
|
result = {
|
|
131
|
-
f"{self.name}.auc": self._score_split(
|
|
155
|
+
f"{self.name}.auc": self._score_split(
|
|
156
|
+
X, X_syn, discrete_features, self.random_state
|
|
157
|
+
),
|
|
132
158
|
}
|
|
133
159
|
|
|
134
160
|
if X_test is not None:
|
|
@@ -146,7 +172,12 @@ class ClassifierTwoSampleTest(BaseMetric):
|
|
|
146
172
|
random_state=self.random_state + repeat,
|
|
147
173
|
)
|
|
148
174
|
test_scores.append(
|
|
149
|
-
self._score_split(
|
|
175
|
+
self._score_split(
|
|
176
|
+
X_test,
|
|
177
|
+
syn_sample,
|
|
178
|
+
discrete_features,
|
|
179
|
+
self.random_state + repeat,
|
|
180
|
+
)
|
|
150
181
|
)
|
|
151
182
|
result[f"{self.name}.test.auc"] = float(np.mean(test_scores))
|
|
152
183
|
|
|
@@ -157,12 +188,64 @@ class ClassifierTwoSampleTest(BaseMetric):
|
|
|
157
188
|
real: pd.DataFrame,
|
|
158
189
|
syn: pd.DataFrame,
|
|
159
190
|
discrete_features: list,
|
|
191
|
+
random_state: int,
|
|
160
192
|
) -> float:
|
|
161
193
|
x_train = pd.concat([real, syn], ignore_index=True).copy()
|
|
162
194
|
y_train = pd.concat(
|
|
163
195
|
[pd.Series([0] * len(real)), pd.Series([1] * len(syn))],
|
|
164
196
|
ignore_index=True,
|
|
165
197
|
)
|
|
198
|
+
if self.scoring in {"holdout", "cv_holdout"}:
|
|
199
|
+
x_train, x_test, y_train, y_test = train_test_split(
|
|
200
|
+
x_train,
|
|
201
|
+
y_train,
|
|
202
|
+
test_size=0.2,
|
|
203
|
+
stratify=y_train,
|
|
204
|
+
random_state=random_state,
|
|
205
|
+
)
|
|
206
|
+
x_val = y_val = None
|
|
207
|
+
model_params = self.model_params.copy()
|
|
208
|
+
if self.scoring == "cv_holdout":
|
|
209
|
+
cv_result = cv_ml_task(
|
|
210
|
+
x_train,
|
|
211
|
+
y_train,
|
|
212
|
+
discrete_features,
|
|
213
|
+
task="binary",
|
|
214
|
+
model_name=self.model_name,
|
|
215
|
+
model_params=model_params,
|
|
216
|
+
random_state=random_state,
|
|
217
|
+
score_fn="auc",
|
|
218
|
+
nfold=self.nfold,
|
|
219
|
+
)
|
|
220
|
+
model_params["n_estimators"] = cv_result["n_estimators"]
|
|
221
|
+
model_params.pop("early_stopping_rounds")
|
|
222
|
+
model_params.pop("num_boost_round", None)
|
|
223
|
+
elif (
|
|
224
|
+
resolve_model_name(self.model_name) == "xgboost"
|
|
225
|
+
and self.model_params.get("early_stopping_rounds") is not None
|
|
226
|
+
):
|
|
227
|
+
x_train, x_val, y_train, y_val = train_test_split(
|
|
228
|
+
x_train,
|
|
229
|
+
y_train,
|
|
230
|
+
test_size=1 / 8,
|
|
231
|
+
stratify=y_train,
|
|
232
|
+
random_state=random_state,
|
|
233
|
+
)
|
|
234
|
+
return ml_task(
|
|
235
|
+
x_train,
|
|
236
|
+
x_test,
|
|
237
|
+
y_train,
|
|
238
|
+
y_test,
|
|
239
|
+
discrete_features,
|
|
240
|
+
task="binary",
|
|
241
|
+
model_name=self.model_name,
|
|
242
|
+
model_params=model_params,
|
|
243
|
+
random_state=random_state,
|
|
244
|
+
score_fns=["auc"],
|
|
245
|
+
X_val=x_val,
|
|
246
|
+
y_val=y_val,
|
|
247
|
+
)["auc"]
|
|
248
|
+
|
|
166
249
|
minority_class_size = min(len(real), len(syn))
|
|
167
250
|
if minority_class_size < 2:
|
|
168
251
|
raise ValueError(
|
|
@@ -178,7 +261,7 @@ class ClassifierTwoSampleTest(BaseMetric):
|
|
|
178
261
|
task="binary",
|
|
179
262
|
model_name=self.model_name,
|
|
180
263
|
model_params=self.model_params,
|
|
181
|
-
random_state=
|
|
264
|
+
random_state=random_state,
|
|
182
265
|
score_fn="auc",
|
|
183
266
|
nfold=nfold,
|
|
184
267
|
)["auc"]
|
|
@@ -3,8 +3,10 @@ import warnings
|
|
|
3
3
|
from functools import lru_cache
|
|
4
4
|
|
|
5
5
|
import numpy as np
|
|
6
|
+
import torch
|
|
6
7
|
import pandas as pd
|
|
7
8
|
import xgboost as xgb
|
|
9
|
+
import sdm
|
|
8
10
|
from sklearn.compose import ColumnTransformer
|
|
9
11
|
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
|
|
10
12
|
from sklearn.linear_model import ElasticNet, LogisticRegression
|
|
@@ -126,18 +128,16 @@ def cv_ml_task(
|
|
|
126
128
|
key = score_key(resolve_score_fn(score_fn)[0])
|
|
127
129
|
|
|
128
130
|
if model_name == "xgboost":
|
|
129
|
-
return
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
)
|
|
140
|
-
}
|
|
131
|
+
return _xgboost_cv_score(
|
|
132
|
+
X,
|
|
133
|
+
y,
|
|
134
|
+
categorical_features,
|
|
135
|
+
task,
|
|
136
|
+
key,
|
|
137
|
+
model_params,
|
|
138
|
+
random_state,
|
|
139
|
+
nfold,
|
|
140
|
+
)
|
|
141
141
|
|
|
142
142
|
model = Pipeline(
|
|
143
143
|
[
|
|
@@ -218,23 +218,33 @@ def ml_task(
|
|
|
218
218
|
X_val = X_val.loc[target_info["val_mask"]]
|
|
219
219
|
y_val = y_val.loc[target_info["val_mask"]]
|
|
220
220
|
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
221
|
+
if model_name != "tfm":
|
|
222
|
+
X_train, X_test, X_val = _transform_features(
|
|
223
|
+
model_name, categorical_features, numerical_features, X_train, X_test, X_val
|
|
224
|
+
)
|
|
224
225
|
y_train, y_test, y_val = _transform_target(
|
|
225
226
|
y_train, y_test, y_val, task, target_info
|
|
226
227
|
)
|
|
227
|
-
|
|
228
228
|
if model_name == "xgboost":
|
|
229
229
|
_maybe_enable_xgboost_gpu(model_params, *X_train.shape)
|
|
230
|
+
if model_name == "tfm":
|
|
231
|
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
232
|
+
model_params["device"] = device
|
|
233
|
+
|
|
234
|
+
# create required tensor inputs, since preprocessing is skipped, X_train is still a pandas DataFrame
|
|
235
|
+
X_train, y_train = _to_table_tensor(X_train, y_train, categorical_features, numerical_features, task, device)
|
|
236
|
+
X_test, _ = _to_table_tensor(X_test, None, categorical_features, numerical_features, task, device)
|
|
237
|
+
|
|
230
238
|
model = build_ml_model(model_name, task, model_params, random_state=random_state)
|
|
231
239
|
model.fit(
|
|
232
240
|
X_train,
|
|
233
241
|
y_train,
|
|
234
|
-
**_fit_kwargs(model_name, model_params, X_val, y_val),
|
|
242
|
+
**_fit_kwargs(model_name, model_params, X_val, y_val, random_state),
|
|
235
243
|
)
|
|
236
244
|
|
|
237
|
-
preds, pred_labels = _predict(
|
|
245
|
+
preds, pred_labels = _predict(
|
|
246
|
+
model, X_test, y_test, task, target_info, model_name
|
|
247
|
+
)
|
|
238
248
|
if score_fns is None:
|
|
239
249
|
return preds
|
|
240
250
|
return score_ml_predictions(
|
|
@@ -392,24 +402,61 @@ def _transform_target(y_train, y_test, y_val, task, target):
|
|
|
392
402
|
None if y_val is None else target["model_encoder"].transform(y_val),
|
|
393
403
|
)
|
|
394
404
|
|
|
405
|
+
def _to_table_tensor(X, y, categorical_features, numerical_features, task, device):
|
|
406
|
+
stypes = {c: "categorical" for c in categorical_features} | {
|
|
407
|
+
c: "numerical" for c in numerical_features
|
|
408
|
+
}
|
|
409
|
+
x_t = sdm.TableTensor.from_pandas(X[list(stypes)], stypes, device=device)
|
|
410
|
+
if y is None:
|
|
411
|
+
return x_t, None
|
|
412
|
+
y_stype = "numerical" if task == "regression" else "categorical"
|
|
413
|
+
y_df = pd.DataFrame({"target": np.asarray(y)})
|
|
414
|
+
y_t = sdm.TableTensor.from_pandas(y_df, {"target": y_stype}, device=device)
|
|
415
|
+
return x_t, y_t
|
|
416
|
+
|
|
395
417
|
|
|
396
|
-
def _fit_kwargs(model_name, model_params, X_val, y_val):
|
|
418
|
+
def _fit_kwargs(model_name, model_params, X_val, y_val, random_state=0):
|
|
397
419
|
if _uses_xgboost_early_stopping(model_name, model_params):
|
|
398
420
|
return {"eval_set": [(X_val, y_val)]}
|
|
421
|
+
if model_name == "tfm":
|
|
422
|
+
generator = torch.Generator(model_params["device"])
|
|
423
|
+
generator.manual_seed(random_state)
|
|
424
|
+
return {"num_estimators": 8, "generator": generator}
|
|
399
425
|
return {}
|
|
400
426
|
|
|
401
427
|
|
|
402
|
-
def
|
|
428
|
+
def _predict_tfm(model, X_test, task):
|
|
429
|
+
"""Run KumoTabular and return numpy outputs.
|
|
430
|
+
|
|
431
|
+
Classification: (class-probability matrix, class labels per column), where
|
|
432
|
+
the labels are the (encoded) classes seen in the context rows.
|
|
433
|
+
Regression: the mean of the 999 equally spaced predicted quantiles
|
|
434
|
+
(q001..q999), which approximates the mean of the predictive distribution.
|
|
435
|
+
"""
|
|
436
|
+
out = model.predict(X_test).to_pandas()
|
|
403
437
|
if task == "regression":
|
|
438
|
+
return out.to_numpy(dtype=float).mean(axis=1), None
|
|
439
|
+
return out.to_numpy(dtype=float), out.columns.to_numpy(dtype=int)
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _predict(model, X_test, y_test, task, target, model_name=None):
|
|
443
|
+
if model_name == "tfm":
|
|
444
|
+
raw_preds, model_classes = _predict_tfm(model, X_test, task)
|
|
445
|
+
if task == "regression":
|
|
446
|
+
return raw_preds, raw_preds
|
|
447
|
+
pred_labels_idx = model_classes[raw_preds.argmax(axis=1)]
|
|
448
|
+
elif task == "regression":
|
|
404
449
|
preds = model.predict(X_test)
|
|
405
450
|
return preds, preds
|
|
451
|
+
else:
|
|
452
|
+
raw_preds = model.predict_proba(X_test)
|
|
453
|
+
model_classes = np.asarray(
|
|
454
|
+
getattr(model, "classes_", np.arange(raw_preds.shape[1])), dtype=int
|
|
455
|
+
)
|
|
456
|
+
pred_labels_idx = np.asarray(model.predict(X_test), dtype=int)
|
|
406
457
|
|
|
407
|
-
raw_preds = model.predict_proba(X_test)
|
|
408
|
-
model_classes = np.asarray(
|
|
409
|
-
getattr(model, "classes_", np.arange(raw_preds.shape[1])), dtype=int
|
|
410
|
-
)
|
|
411
458
|
class_map = target["class_map"]
|
|
412
|
-
pred_labels = class_map[
|
|
459
|
+
pred_labels = class_map[pred_labels_idx]
|
|
413
460
|
|
|
414
461
|
if task == "multiclass" or target["score_task"] == "multiclass":
|
|
415
462
|
preds = np.zeros((len(y_test), len(target["score_encoder"].classes_)))
|
|
@@ -473,7 +520,11 @@ def _xgboost_cv_score(
|
|
|
473
520
|
maximize=maximize,
|
|
474
521
|
as_pandas=True,
|
|
475
522
|
)
|
|
476
|
-
|
|
523
|
+
scores = results[f"test-{metric}-mean"]
|
|
524
|
+
return {
|
|
525
|
+
score_key_: float(scores.iloc[-1]),
|
|
526
|
+
"n_estimators": int(scores.argmax() if maximize else scores.argmin()) + 1,
|
|
527
|
+
}
|
|
477
528
|
|
|
478
529
|
|
|
479
530
|
def _xgboost_cv_metric(score_key_: str):
|
|
@@ -558,6 +609,8 @@ def resolve_model_name(model_name: str):
|
|
|
558
609
|
return "svm"
|
|
559
610
|
if model_name.startswith("xgb"):
|
|
560
611
|
return "xgboost"
|
|
612
|
+
if model_name in ["kumo", "tfm", "fm"]:
|
|
613
|
+
return "tfm"
|
|
561
614
|
raise ValueError(f"Model {model_name} not supported")
|
|
562
615
|
|
|
563
616
|
|
|
@@ -580,6 +633,13 @@ def build_ml_model(
|
|
|
580
633
|
),
|
|
581
634
|
"tree_method": "hist",
|
|
582
635
|
}
|
|
636
|
+
elif model_name == "tfm":
|
|
637
|
+
model = sdm.models.KumoTabular
|
|
638
|
+
# sdm only knows "classification" and "regression" (binary/multiclass are both "classification")
|
|
639
|
+
defaults = {
|
|
640
|
+
"task": "regression" if task == "regression" else "classification",
|
|
641
|
+
"size": "small",
|
|
642
|
+
}
|
|
583
643
|
elif model_name == "randomforest":
|
|
584
644
|
model = (
|
|
585
645
|
RandomForestRegressor if task == "regression" else RandomForestClassifier
|
|
@@ -63,6 +63,7 @@ class DCR(BaseMetric):
|
|
|
63
63
|
Args:
|
|
64
64
|
subsample_test_size (bool): Whether to subsample the training set and synthetic set to the test set size. Prevents biasing DCR due to different sample sizes. If used, multiple iterations average random train/synthetic subsamples of size ``|X_test|``. Default: True.
|
|
65
65
|
random_state (int): Random seed for reproducibility. Default: 0.
|
|
66
|
+
repeats (int): Number of repeated evaluations when subsampling. Default: 5.
|
|
66
67
|
|
|
67
68
|
Outputs:
|
|
68
69
|
DCR score, nearest-neighbor direction proportions, threshold risks,
|
|
@@ -93,9 +94,11 @@ class DCR(BaseMetric):
|
|
|
93
94
|
self,
|
|
94
95
|
subsample_test_size: bool = True,
|
|
95
96
|
random_state: int = 0,
|
|
97
|
+
repeats: int = 5,
|
|
96
98
|
):
|
|
97
99
|
super().__init__(random_state=random_state)
|
|
98
100
|
self.subsample_test_size = subsample_test_size
|
|
101
|
+
self.repeats = repeats
|
|
99
102
|
|
|
100
103
|
def _evaluate(
|
|
101
104
|
self,
|
|
@@ -139,7 +142,7 @@ class DCR(BaseMetric):
|
|
|
139
142
|
"DCR with NNDR requires at least two train rows and two test rows "
|
|
140
143
|
"after applying subsampling."
|
|
141
144
|
)
|
|
142
|
-
num_iterations =
|
|
145
|
+
num_iterations = self.repeats if self.subsample_test_size else 1
|
|
143
146
|
metric_values = {}
|
|
144
147
|
|
|
145
148
|
rng = np.random.default_rng(self.random_state)
|
|
@@ -2,6 +2,7 @@ import pickle
|
|
|
2
2
|
import importlib
|
|
3
3
|
import inspect
|
|
4
4
|
import re
|
|
5
|
+
from time import monotonic
|
|
5
6
|
from copy import deepcopy
|
|
6
7
|
from abc import ABC, abstractmethod
|
|
7
8
|
from pathlib import Path
|
|
@@ -46,6 +47,8 @@ class BaseGenerator(ABC):
|
|
|
46
47
|
self.full_determinism = full_determinism
|
|
47
48
|
self.ordinal_encoder = None
|
|
48
49
|
self._base_discrete_features = []
|
|
50
|
+
self.gross_training_time_ = None
|
|
51
|
+
self.net_training_time_ = None
|
|
49
52
|
if torch.cuda.is_available():
|
|
50
53
|
torch.set_float32_matmul_precision("high")
|
|
51
54
|
|
|
@@ -95,18 +98,34 @@ class BaseGenerator(ABC):
|
|
|
95
98
|
Returns:
|
|
96
99
|
The fitted generator.
|
|
97
100
|
"""
|
|
98
|
-
|
|
99
|
-
self.
|
|
100
|
-
self.schema = TabularSchema.from_dataframe(X)
|
|
101
|
-
X = self._fit_encode_categoricals(X, discrete_features)
|
|
102
|
-
set_seed(self.random_state, self.full_determinism)
|
|
103
|
-
self._generate_model_space = True
|
|
101
|
+
train_start = monotonic()
|
|
102
|
+
self.net_training_time_ = None
|
|
104
103
|
try:
|
|
104
|
+
discrete_features = list(discrete_features)
|
|
105
|
+
self._validate_feature_mix(X, discrete_features)
|
|
106
|
+
self.schema = TabularSchema.from_dataframe(X)
|
|
107
|
+
X = self._fit_encode_categoricals(X, discrete_features)
|
|
108
|
+
set_seed(self.random_state, self.full_determinism)
|
|
109
|
+
self._generate_model_space = True
|
|
105
110
|
self._fit(X, discrete_features)
|
|
106
111
|
finally:
|
|
107
112
|
self._generate_model_space = False
|
|
113
|
+
self.gross_training_time_ = monotonic() - train_start
|
|
114
|
+
if self.net_training_time_ is None:
|
|
115
|
+
self.net_training_time_ = self.gross_training_time_
|
|
108
116
|
return self
|
|
109
117
|
|
|
118
|
+
def get_gross_training_time(self) -> float:
|
|
119
|
+
"""Return seconds spent in the latest fit, including setup and validation."""
|
|
120
|
+
return self.gross_training_time_
|
|
121
|
+
|
|
122
|
+
def get_net_training_time(self) -> float:
|
|
123
|
+
"""Return training-step seconds, excluding validation, for deep models.
|
|
124
|
+
|
|
125
|
+
Other generators return the full fit duration.
|
|
126
|
+
"""
|
|
127
|
+
return self.net_training_time_
|
|
128
|
+
|
|
110
129
|
def _fit_encode_categoricals(
|
|
111
130
|
self, X: pd.DataFrame, discrete_features: list
|
|
112
131
|
) -> pd.DataFrame:
|
|
@@ -200,6 +219,8 @@ class BaseGenerator(ABC):
|
|
|
200
219
|
"full_determinism": self.full_determinism,
|
|
201
220
|
"ordinal_encoder": self.ordinal_encoder,
|
|
202
221
|
"_base_discrete_features": self._base_discrete_features,
|
|
222
|
+
"gross_training_time_": self.gross_training_time_,
|
|
223
|
+
"net_training_time_": self.net_training_time_,
|
|
203
224
|
}
|
|
204
225
|
|
|
205
226
|
def _save_extra(self, path: Path) -> None:
|
|
@@ -298,7 +298,7 @@ class CDTDGenerator(BaseGenerator):
|
|
|
298
298
|
self.diff_model.train()
|
|
299
299
|
return bad_val_steps >= self.patience
|
|
300
300
|
|
|
301
|
-
train_time = 0.0
|
|
301
|
+
train_time = self.net_training_time_ = 0.0
|
|
302
302
|
timed_out = False
|
|
303
303
|
self.trained_steps_ = 0
|
|
304
304
|
with tqdm(initial=current_step, total=self.training_steps) as pbar:
|
|
@@ -335,6 +335,7 @@ class CDTDGenerator(BaseGenerator):
|
|
|
335
335
|
param_group["lr"] = scheduler(current_step)
|
|
336
336
|
|
|
337
337
|
train_time += time.monotonic() - step_start_time
|
|
338
|
+
self.net_training_time_ = train_time
|
|
338
339
|
if self.cap_train_time is not None and train_time > self.cap_train_time:
|
|
339
340
|
print(f"Training timed out after {self.cap_train_time} seconds.")
|
|
340
341
|
timed_out = True
|
|
@@ -348,7 +349,7 @@ class CDTDGenerator(BaseGenerator):
|
|
|
348
349
|
if validate():
|
|
349
350
|
break
|
|
350
351
|
|
|
351
|
-
if timed_out and
|
|
352
|
+
if timed_out and best_val_model is not None:
|
|
352
353
|
validate()
|
|
353
354
|
|
|
354
355
|
if best_val_model is None:
|
|
@@ -39,6 +39,50 @@ DEFAULT_CONFIG = {
|
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
NETWORK_SIZE_CONFIGS = {
|
|
42
|
+
"forestdiffusion": {
|
|
43
|
+
"vp": {
|
|
44
|
+
"num_timesteps": 50,
|
|
45
|
+
"diffusion_type": "vp",
|
|
46
|
+
"multi_output": False,
|
|
47
|
+
"xgb_hypers": {
|
|
48
|
+
"n_estimators": 100,
|
|
49
|
+
"max_depth": 7,
|
|
50
|
+
"reg_lambda": 0.0,
|
|
51
|
+
"early_stopping_rounds": 20,
|
|
52
|
+
"n_jobs": 1,
|
|
53
|
+
},
|
|
54
|
+
"noise_samples_per_row": 50,
|
|
55
|
+
"eps": 0.001,
|
|
56
|
+
"beta_min": 0.1,
|
|
57
|
+
"beta_max": 8,
|
|
58
|
+
"n_jobs": -1,
|
|
59
|
+
"backend": "loky",
|
|
60
|
+
"n_batch": -1,
|
|
61
|
+
"memory_cap_gb": 128,
|
|
62
|
+
"conditional_generation": False,
|
|
63
|
+
},
|
|
64
|
+
"flow": {
|
|
65
|
+
"num_timesteps": 50,
|
|
66
|
+
"diffusion_type": "flow",
|
|
67
|
+
"multi_output": False,
|
|
68
|
+
"xgb_hypers": {
|
|
69
|
+
"n_estimators": 100,
|
|
70
|
+
"max_depth": 7,
|
|
71
|
+
"reg_lambda": 0.0,
|
|
72
|
+
"early_stopping_rounds": 20,
|
|
73
|
+
"n_jobs": 1,
|
|
74
|
+
},
|
|
75
|
+
"noise_samples_per_row": 50,
|
|
76
|
+
"eps": 0.0,
|
|
77
|
+
"beta_min": 0.1,
|
|
78
|
+
"beta_max": 8,
|
|
79
|
+
"n_jobs": -1,
|
|
80
|
+
"backend": "loky",
|
|
81
|
+
"n_batch": -1,
|
|
82
|
+
"memory_cap_gb": 128,
|
|
83
|
+
"conditional_generation": False,
|
|
84
|
+
},
|
|
85
|
+
},
|
|
42
86
|
"synthpop": {
|
|
43
87
|
"original": {
|
|
44
88
|
"minibucket": 10,
|
|
@@ -51,6 +51,8 @@ class CTGANGenerator(BaseGenerator):
|
|
|
51
51
|
validation. Default: -1.
|
|
52
52
|
patience (int): Number of consecutive non-improving C2ST validation
|
|
53
53
|
checks before early stopping. Default: 3.
|
|
54
|
+
n_jobs (int): Number of preprocessing workers. -1 uses all CPUs;
|
|
55
|
+
1 runs sequentially. Default: -1.
|
|
54
56
|
max_validation_rows (int): Maximum number of rows reserved for C2ST
|
|
55
57
|
validation. Negative values remove the cap. Extra rows remain in the training set. Default: 30000.
|
|
56
58
|
|
|
@@ -101,6 +103,7 @@ class CTGANGenerator(BaseGenerator):
|
|
|
101
103
|
max_validation_rows: int = 30_000,
|
|
102
104
|
random_state: int = 0,
|
|
103
105
|
full_determinism: bool = False,
|
|
106
|
+
n_jobs: int = -1,
|
|
104
107
|
):
|
|
105
108
|
super().__init__(random_state=random_state, full_determinism=full_determinism)
|
|
106
109
|
require_ctgan()
|
|
@@ -130,6 +133,7 @@ class CTGANGenerator(BaseGenerator):
|
|
|
130
133
|
self.val_steps = val_steps
|
|
131
134
|
self.patience = patience
|
|
132
135
|
self.max_validation_rows = max_validation_rows
|
|
136
|
+
self.n_jobs = n_jobs
|
|
133
137
|
|
|
134
138
|
def _fit(self, X: pd.DataFrame, discrete_features: list):
|
|
135
139
|
from .synthesizer import CTGAN
|
|
@@ -179,6 +183,7 @@ class CTGANGenerator(BaseGenerator):
|
|
|
179
183
|
pac=self.pac,
|
|
180
184
|
cuda=self.cuda,
|
|
181
185
|
cap_train_time=self.cap_train_time,
|
|
186
|
+
n_jobs=self.n_jobs,
|
|
182
187
|
)
|
|
183
188
|
|
|
184
189
|
best_val_score = float("inf")
|
|
@@ -216,7 +221,8 @@ class CTGANGenerator(BaseGenerator):
|
|
|
216
221
|
discrete_features,
|
|
217
222
|
validate_callback=validate_callback,
|
|
218
223
|
)
|
|
219
|
-
|
|
224
|
+
self.net_training_time_ = self.model.net_training_time_
|
|
225
|
+
if getattr(self.model, "timed_out_", False) and best_val_model is not None:
|
|
220
226
|
validate()
|
|
221
227
|
|
|
222
228
|
if best_val_model is not None:
|
|
@@ -264,6 +270,7 @@ class CTGANGenerator(BaseGenerator):
|
|
|
264
270
|
"val_steps": self.val_steps,
|
|
265
271
|
"patience": self.patience,
|
|
266
272
|
"max_validation_rows": self.max_validation_rows,
|
|
273
|
+
"n_jobs": self.n_jobs,
|
|
267
274
|
"discrete_features": getattr(self, "discrete_features", None),
|
|
268
275
|
}
|
|
269
276
|
|