synthyverse 0.2.3__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {synthyverse-0.2.3 → synthyverse-0.2.4}/PKG-INFO +2 -1
- {synthyverse-0.2.3 → synthyverse-0.2.4}/THIRD_PARTY_NOTICES.md +10 -1
- {synthyverse-0.2.3 → synthyverse-0.2.4}/pyproject.toml +2 -1
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/__init__.py +3 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/dgm_utils.py +11 -22
- synthyverse-0.2.4/synthyverse/generators/tabcascade_generator/__init__.py +1 -0
- synthyverse-0.2.4/synthyverse/generators/tabcascade_generator/disttree.py +289 -0
- synthyverse-0.2.4/synthyverse/generators/tabcascade_generator/encoder.py +120 -0
- synthyverse-0.2.4/synthyverse/generators/tabcascade_generator/highres.py +262 -0
- synthyverse-0.2.4/synthyverse/generators/tabcascade_generator/lowres.py +400 -0
- synthyverse-0.2.4/synthyverse/generators/tabcascade_generator/tabcascade.py +644 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabsyn_generator/tabsyn.py +2 -6
- synthyverse-0.2.4/synthyverse/generators/xgbddpm_generator/__init__.py +1 -0
- synthyverse-0.2.4/synthyverse/generators/xgbddpm_generator/xgbddpm.py +580 -0
- synthyverse-0.2.4/synthyverse/generators/xgbdiffusion_generator/__init__.py +1 -0
- synthyverse-0.2.4/synthyverse/generators/xgbdiffusion_generator/xgbdiffusion.py +340 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse.egg-info/PKG-INFO +2 -1
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse.egg-info/SOURCES.txt +10 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse.egg-info/requires.txt +1 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/LICENSE +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/LICENSES/Apache-2.0.txt +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/LICENSES/CTGAN-BSL-1.1.txt +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/LICENSES/MIT.txt +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/MANIFEST.in +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/NOTICE +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/README.md +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/logo/logo.png +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/setup.cfg +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/__init__.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/benchmark/__init__.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/benchmark/synthesis.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/evaluation/__init__.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/evaluation/eval.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/evaluation/fidelity.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/evaluation/gower.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/evaluation/ml.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/evaluation/preprocessing.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/evaluation/privacy.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/evaluation/utility.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/_optional.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/arf_generator/__init__.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/arf_generator/arf.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/arf_generator/model.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/arf_generator/utils.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/base.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/cdtd_generator/__init__.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/cdtd_generator/cdtd.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/cdtd_generator/layers.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/cdtd_generator/utils.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/ctgan_generator/__init__.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/ctgan_generator/ct_gan.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/persistence.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/smote_generator/__init__.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/smote_generator/smote.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/synthpop_generator/__init__.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/synthpop_generator/synthpop.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabddpm_generator/__init__.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabddpm_generator/ddpm_utils.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabddpm_generator/gaussian_multinomial_diffsuion.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabddpm_generator/modules.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabddpm_generator/tabddpm.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabdiff_generator/__init__.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabdiff_generator/diffusion.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabdiff_generator/modules.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabdiff_generator/noise_schedule.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabdiff_generator/tabdiff.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabdiff_generator/transformer.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabsyn_generator/__init__.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabsyn_generator/diffusion.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tabsyn_generator/vae.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tvae_generator/__init__.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/tvae_generator/tvae.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/univariate_generator/__init__.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/generators/univariate_generator/univariate.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/utils/__init__.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/utils/reproducibility.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse/utils/utils.py +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse.egg-info/dependency_links.txt +0 -0
- {synthyverse-0.2.3 → synthyverse-0.2.4}/synthyverse.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: synthyverse
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Summary: Synthetic data generation and evaluation library
|
|
5
5
|
Author: Jim Achterberg, Saif Ul Islam, Zia Ur Rehman
|
|
6
6
|
License: MIT License
|
|
@@ -40,6 +40,7 @@ Requires-Dist: torch
|
|
|
40
40
|
Requires-Dist: torch-ema
|
|
41
41
|
Requires-Dist: einops
|
|
42
42
|
Requires-Dist: imbalanced-learn
|
|
43
|
+
Requires-Dist: xgb-diffusion
|
|
43
44
|
Provides-Extra: ctgan
|
|
44
45
|
Requires-Dist: ctgan; extra == "ctgan"
|
|
45
46
|
Provides-Extra: docs
|
|
@@ -45,10 +45,19 @@ The MIT license text for these adapted implementations is kept in
|
|
|
45
45
|
- Upstream project: `TabDiff (https://github.com/MinkaiXu/TabDiff/)`
|
|
46
46
|
- Upstream license: MIT License
|
|
47
47
|
- Upstream copyright notice: `Copyright 2024 Minkai Xu`
|
|
48
|
-
- Imported or last compared against: `5ecdb33`
|
|
48
|
+
- Imported or last compared against: `Commit 5ecdb33`
|
|
49
49
|
- Modifications: `Refactored API, speed-up masked diffusion by vectorizing loops, fast tensor dataloader.`
|
|
50
50
|
- Upstream NOTICE text, if any: `None`
|
|
51
51
|
|
|
52
|
+
### TabCascadeGenerator
|
|
53
|
+
|
|
54
|
+
- Synthyverse files: `synthyverse/generators/tabcascade_generator/*`
|
|
55
|
+
- Upstream project: `TabCascade (https://github.com/muellermarkus/tabcascade)`
|
|
56
|
+
- Upstream license: MIT License
|
|
57
|
+
- Upstream copyright notice: `Copyright (c) 2026 Markus Mueller`
|
|
58
|
+
- Imported or last compared against: `Commit c8f44bb`
|
|
59
|
+
- Modifications: `Refactored API, simple ordinal encoding for categories.`
|
|
60
|
+
- Upstream NOTICE text, if any: `None`
|
|
52
61
|
|
|
53
62
|
## Apache-2.0-licensed adapted implementations
|
|
54
63
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "synthyverse"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.4"
|
|
8
8
|
description = "Synthetic data generation and evaluation library"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.8"
|
|
@@ -43,6 +43,7 @@ dependencies = [
|
|
|
43
43
|
"torch-ema",
|
|
44
44
|
"einops",
|
|
45
45
|
"imbalanced-learn",
|
|
46
|
+
"xgb-diffusion",
|
|
46
47
|
]
|
|
47
48
|
|
|
48
49
|
[tool.setuptools]
|
|
@@ -16,9 +16,12 @@ _BASE_GENERATORS = {
|
|
|
16
16
|
"CDTDGenerator": (".cdtd_generator", "cdtd"),
|
|
17
17
|
"TabDDPMGenerator": (".tabddpm_generator", "tabddpm"),
|
|
18
18
|
"TabDiffGenerator": (".tabdiff_generator", "tabdiff"),
|
|
19
|
+
"TabCascadeGenerator": (".tabcascade_generator", "tabcascade"),
|
|
19
20
|
"UnivariateGenerator": (".univariate_generator", "univariate"),
|
|
20
21
|
"SMOTEGenerator": (".smote_generator", "smote"),
|
|
21
22
|
"SynthpopGenerator": (".synthpop_generator", "synthpop"),
|
|
23
|
+
"XGBDDPMGenerator": (".xgbddpm_generator", "xgbddpm"),
|
|
24
|
+
"XGBDiffusionGenerator": (".xgbdiffusion_generator", "xgbdiffusion"),
|
|
22
25
|
}
|
|
23
26
|
|
|
24
27
|
_CTGAN_GENERATORS = {
|
|
@@ -55,15 +55,12 @@ class MLPDiffusion(nn.Module):
|
|
|
55
55
|
|
|
56
56
|
|
|
57
57
|
class FastTensorDataLoader:
|
|
58
|
-
"""Iterate over
|
|
58
|
+
"""Iterate over tensors in mini-batches."""
|
|
59
59
|
|
|
60
|
-
def __init__(self,
|
|
61
|
-
self.dataset_len =
|
|
62
|
-
assert all(
|
|
63
|
-
|
|
64
|
-
)
|
|
65
|
-
self.X_cat = X_cat
|
|
66
|
-
self.X_cont = X_cont
|
|
60
|
+
def __init__(self, *data, batch_size=32, shuffle=False, drop_last=False):
|
|
61
|
+
self.dataset_len = next(t.shape[0] for t in data if t is not None)
|
|
62
|
+
assert all(t.shape[0] == self.dataset_len for t in data if t is not None)
|
|
63
|
+
self.data = data
|
|
67
64
|
self.batch_size = batch_size
|
|
68
65
|
self.shuffle = shuffle
|
|
69
66
|
|
|
@@ -84,22 +81,14 @@ class FastTensorDataLoader:
|
|
|
84
81
|
|
|
85
82
|
if self.indices is not None:
|
|
86
83
|
indices = self.indices[self.i : self.i + self.batch_size]
|
|
87
|
-
batch = (
|
|
88
|
-
torch.index_select(
|
|
89
|
-
|
|
90
|
-
else None,
|
|
91
|
-
torch.index_select(self.X_cont, 0, indices)
|
|
92
|
-
if self.X_cont is not None
|
|
93
|
-
else None,
|
|
84
|
+
batch = tuple(
|
|
85
|
+
torch.index_select(t, 0, indices) if t is not None else None
|
|
86
|
+
for t in self.data
|
|
94
87
|
)
|
|
95
88
|
else:
|
|
96
|
-
batch = (
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
else None,
|
|
100
|
-
self.X_cont[self.i : self.i + self.batch_size]
|
|
101
|
-
if self.X_cont is not None
|
|
102
|
-
else None,
|
|
89
|
+
batch = tuple(
|
|
90
|
+
t[self.i : self.i + self.batch_size] if t is not None else None
|
|
91
|
+
for t in self.data
|
|
103
92
|
)
|
|
104
93
|
|
|
105
94
|
self.i += self.batch_size
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .tabcascade import TabCascadeGenerator
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
import math
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
6
|
+
from scipy import stats
|
|
7
|
+
from scipy.optimize import minimize
|
|
8
|
+
from scipy.stats import chi2
|
|
9
|
+
from sklearn.preprocessing import OrdinalEncoder
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class GaussianFamily:
|
|
13
|
+
npar = 2
|
|
14
|
+
|
|
15
|
+
def start_params(self, y, weights=None):
|
|
16
|
+
if weights is None:
|
|
17
|
+
mu = float(np.mean(y))
|
|
18
|
+
sigma = float(np.std(y, ddof=1)) or 1.0
|
|
19
|
+
else:
|
|
20
|
+
w = weights / weights.sum()
|
|
21
|
+
mu = float(np.dot(w, y))
|
|
22
|
+
sigma = float(np.sqrt(np.dot(w, (y - mu) ** 2))) or 1.0
|
|
23
|
+
return np.array([mu, np.log(sigma)])
|
|
24
|
+
|
|
25
|
+
def inverse_link(self, eta):
|
|
26
|
+
return np.asarray(eta, dtype=float).copy()
|
|
27
|
+
|
|
28
|
+
def link_bounds(self):
|
|
29
|
+
return [(None, None), (-100.0, None)]
|
|
30
|
+
|
|
31
|
+
def log_likelihood(self, y, params, weights=None):
|
|
32
|
+
mu, log_sigma = float(params[0]), float(params[1])
|
|
33
|
+
weights = np.ones(len(y)) if weights is None else np.asarray(weights, dtype=float)
|
|
34
|
+
ll = (
|
|
35
|
+
-0.5 * np.log(2.0 * np.pi)
|
|
36
|
+
- log_sigma
|
|
37
|
+
- 0.5 * (y - mu) ** 2 * np.exp(-2.0 * log_sigma)
|
|
38
|
+
)
|
|
39
|
+
return float(np.dot(weights, ll))
|
|
40
|
+
|
|
41
|
+
def score(self, y, params, weights=None):
|
|
42
|
+
mu, log_sigma = float(params[0]), float(params[1])
|
|
43
|
+
weights = np.ones(len(y)) if weights is None else np.asarray(weights, dtype=float)
|
|
44
|
+
residual = y - mu
|
|
45
|
+
exp_neg2 = np.exp(-2.0 * log_sigma)
|
|
46
|
+
score = np.column_stack(
|
|
47
|
+
[residual * exp_neg2, residual**2 * exp_neg2 - 1.0]
|
|
48
|
+
)
|
|
49
|
+
return score * weights[:, np.newaxis]
|
|
50
|
+
|
|
51
|
+
def hessian(self, y, params, weights=None):
|
|
52
|
+
mu, log_sigma = float(params[0]), float(params[1])
|
|
53
|
+
weights = np.ones(len(y)) if weights is None else np.asarray(weights, dtype=float)
|
|
54
|
+
residual = y - mu
|
|
55
|
+
exp_neg2 = np.exp(-2.0 * log_sigma)
|
|
56
|
+
return np.array(
|
|
57
|
+
[
|
|
58
|
+
[-weights.sum() * exp_neg2, -2.0 * np.dot(weights, residual) * exp_neg2],
|
|
59
|
+
[
|
|
60
|
+
-2.0 * np.dot(weights, residual) * exp_neg2,
|
|
61
|
+
-2.0 * np.dot(weights, residual**2) * exp_neg2,
|
|
62
|
+
],
|
|
63
|
+
]
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
def to_user_params(self, params):
|
|
67
|
+
return np.array([float(params[0]), np.exp(float(params[1]))]), ["mu", "sigma"]
|
|
68
|
+
|
|
69
|
+
def mean(self, params):
|
|
70
|
+
return float(params[0])
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class DistFit:
|
|
74
|
+
def __init__(self, family=None):
|
|
75
|
+
self.family = family or GaussianFamily()
|
|
76
|
+
|
|
77
|
+
def fit(self, y, sample_weight=None):
|
|
78
|
+
y = np.asarray(y, dtype=float)
|
|
79
|
+
weights = np.ones(len(y)) if sample_weight is None else np.asarray(sample_weight, dtype=float)
|
|
80
|
+
eta0 = self.family.start_params(y, weights)
|
|
81
|
+
|
|
82
|
+
def neg_loglik(eta):
|
|
83
|
+
return -self.family.log_likelihood(y, self.family.inverse_link(eta), weights)
|
|
84
|
+
|
|
85
|
+
def neg_score_eta(eta):
|
|
86
|
+
params = self.family.inverse_link(eta)
|
|
87
|
+
return -self.family.score(y, params, weights).sum(axis=0)
|
|
88
|
+
|
|
89
|
+
result = minimize(
|
|
90
|
+
neg_loglik,
|
|
91
|
+
eta0,
|
|
92
|
+
jac=neg_score_eta,
|
|
93
|
+
method="L-BFGS-B",
|
|
94
|
+
bounds=self.family.link_bounds(),
|
|
95
|
+
options={"maxiter": 500, "ftol": 1e-12, "gtol": 1e-8},
|
|
96
|
+
)
|
|
97
|
+
params = self.family.inverse_link(result.x)
|
|
98
|
+
display, names = self.family.to_user_params(params)
|
|
99
|
+
self.coef_ = {name: float(value) for name, value in zip(names, display)}
|
|
100
|
+
self.coef_array_ = display.copy()
|
|
101
|
+
self.loglik_ = float(-result.fun)
|
|
102
|
+
self.score_matrix_ = self.family.score(y, params, weights)
|
|
103
|
+
self.hessian_ = self.family.hessian(y, params, weights)
|
|
104
|
+
self.estfun_ = self.score_matrix_
|
|
105
|
+
return self
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _quadratic_stat(t, mu, sigma):
|
|
109
|
+
diff = t - mu
|
|
110
|
+
eigvals, eigvecs = np.linalg.eigh(sigma)
|
|
111
|
+
tol = max(1e-14, 1e-10 * float(np.abs(eigvals).max() or 1.0))
|
|
112
|
+
pos = eigvals > tol
|
|
113
|
+
if not pos.any():
|
|
114
|
+
return 0.0, 0
|
|
115
|
+
sigma_pinv = eigvecs[:, pos] @ np.diag(1.0 / eigvals[pos]) @ eigvecs[:, pos].T
|
|
116
|
+
return float(diff @ sigma_pinv @ diff), int(pos.sum())
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def select_variable(X, distfit, alpha=0.05):
|
|
120
|
+
h = distfit.estfun_
|
|
121
|
+
n, p_vars = X.shape
|
|
122
|
+
p_values = np.ones(p_vars)
|
|
123
|
+
for j in range(p_vars):
|
|
124
|
+
g = X[:, j]
|
|
125
|
+
h_bar = h.mean(axis=0)
|
|
126
|
+
h_c = h - h_bar
|
|
127
|
+
v_h = (h_c.T @ h_c) / n
|
|
128
|
+
sigma_g = (n / (n - 1)) * float(g @ g) - (1 / (n - 1)) * float(g.sum()) ** 2
|
|
129
|
+
stat, df = _quadratic_stat(g @ h, g.sum() * h_bar, sigma_g * v_h)
|
|
130
|
+
p_values[j] = float(chi2.sf(stat, df=df)) if df > 0 else 1.0
|
|
131
|
+
j_star = int(np.argmin(p_values))
|
|
132
|
+
return j_star if min(p_vars * float(p_values[j_star]), 1.0) < alpha else None
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def find_split(x, distfit, minbucket):
|
|
136
|
+
h = distfit.estfun_
|
|
137
|
+
n = h.shape[0]
|
|
138
|
+
order = np.argsort(x, kind="stable")
|
|
139
|
+
x_sorted = x[order]
|
|
140
|
+
h_sorted = h[order]
|
|
141
|
+
h_bar = h.mean(axis=0)
|
|
142
|
+
h_c = h - h_bar
|
|
143
|
+
v_h = (h_c.T @ h_c) / n
|
|
144
|
+
eigvals, eigvecs = np.linalg.eigh(v_h)
|
|
145
|
+
tol = max(1e-14, 1e-10 * float(np.abs(eigvals).max() or 1.0))
|
|
146
|
+
pos = eigvals > tol
|
|
147
|
+
if not pos.any():
|
|
148
|
+
return None
|
|
149
|
+
v_h_pinv = eigvecs[:, pos] @ np.diag(1.0 / eigvals[pos]) @ eigvecs[:, pos].T
|
|
150
|
+
h_cumsum = np.cumsum(h_sorted, axis=0)
|
|
151
|
+
best_stat = -np.inf
|
|
152
|
+
best_k = None
|
|
153
|
+
for k in range(minbucket, n - minbucket + 1):
|
|
154
|
+
if x_sorted[k - 1] == x_sorted[k]:
|
|
155
|
+
continue
|
|
156
|
+
diff = h_cumsum[k - 1] - k * h_bar
|
|
157
|
+
sigma_g = k * (n - k) / (n - 1)
|
|
158
|
+
stat = float(diff @ v_h_pinv @ diff) / sigma_g
|
|
159
|
+
if stat > best_stat:
|
|
160
|
+
best_stat = stat
|
|
161
|
+
best_k = k
|
|
162
|
+
return None if best_k is None else float(x_sorted[best_k - 1])
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@dataclass
|
|
166
|
+
class Node:
|
|
167
|
+
node_id: int
|
|
168
|
+
depth: int
|
|
169
|
+
n_samples: int
|
|
170
|
+
distfit: DistFit
|
|
171
|
+
is_leaf: bool = True
|
|
172
|
+
split_var: int = None
|
|
173
|
+
split_val: float = None
|
|
174
|
+
left: "Node" = None
|
|
175
|
+
right: "Node" = None
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
class DistTreeRegressor:
|
|
179
|
+
def __init__(self, max_depth=None, alpha=0.05, minprob=0.01):
|
|
180
|
+
self.family = GaussianFamily()
|
|
181
|
+
self.max_depth = max_depth
|
|
182
|
+
self.alpha = alpha
|
|
183
|
+
self.minprob = minprob
|
|
184
|
+
|
|
185
|
+
def fit(self, X, y):
|
|
186
|
+
X = np.asarray(X, dtype=float)
|
|
187
|
+
y = np.asarray(y, dtype=float).ravel()
|
|
188
|
+
self._node_counter = 0
|
|
189
|
+
self.tree_ = self._grow(np.arange(len(y)), X, y, 0, math.ceil(10 * self.family.npar))
|
|
190
|
+
return self
|
|
191
|
+
|
|
192
|
+
def _next_id(self):
|
|
193
|
+
node_id = self._node_counter
|
|
194
|
+
self._node_counter += 1
|
|
195
|
+
return node_id
|
|
196
|
+
|
|
197
|
+
def _grow(self, indices, X, y, depth, min_leaf):
|
|
198
|
+
y_node = y[indices]
|
|
199
|
+
if float(np.var(y_node)) < 1e-14:
|
|
200
|
+
return Node(self._next_id(), depth, len(indices), DistFit(self.family).fit(y_node))
|
|
201
|
+
|
|
202
|
+
distfit = DistFit(self.family).fit(y_node)
|
|
203
|
+
node = Node(self._next_id(), depth, len(indices), distfit)
|
|
204
|
+
if (
|
|
205
|
+
not np.all(np.isfinite(distfit.estfun_))
|
|
206
|
+
or (self.max_depth is not None and depth >= self.max_depth)
|
|
207
|
+
or len(indices) < min_leaf
|
|
208
|
+
):
|
|
209
|
+
return node
|
|
210
|
+
|
|
211
|
+
X_node = X[indices]
|
|
212
|
+
split_var = select_variable(X_node, distfit, self.alpha)
|
|
213
|
+
if split_var is None:
|
|
214
|
+
return node
|
|
215
|
+
threshold = find_split(
|
|
216
|
+
X_node[:, split_var],
|
|
217
|
+
distfit,
|
|
218
|
+
max(min_leaf, math.ceil(self.minprob * len(indices))),
|
|
219
|
+
)
|
|
220
|
+
if threshold is None:
|
|
221
|
+
return node
|
|
222
|
+
|
|
223
|
+
left_mask = X_node[:, split_var] <= threshold
|
|
224
|
+
node.is_leaf = False
|
|
225
|
+
node.split_var = split_var
|
|
226
|
+
node.split_val = threshold
|
|
227
|
+
node.left = self._grow(indices[left_mask], X, y, depth + 1, min_leaf)
|
|
228
|
+
node.right = self._grow(indices[~left_mask], X, y, depth + 1, min_leaf)
|
|
229
|
+
return node
|
|
230
|
+
|
|
231
|
+
def _apply_one(self, x):
|
|
232
|
+
node = self.tree_
|
|
233
|
+
while not node.is_leaf:
|
|
234
|
+
node = node.left if x[node.split_var] <= node.split_val else node.right
|
|
235
|
+
return node
|
|
236
|
+
|
|
237
|
+
def predict_params(self, X):
|
|
238
|
+
leaves = [self._apply_one(row) for row in np.asarray(X, dtype=float)]
|
|
239
|
+
return {
|
|
240
|
+
"mu": np.array([leaf.distfit.coef_["mu"] for leaf in leaves]),
|
|
241
|
+
"sigma": np.array([leaf.distfit.coef_["sigma"] for leaf in leaves]),
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
class DistTree:
|
|
246
|
+
def __init__(self, max_depth=3):
|
|
247
|
+
self.max_depth = max_depth
|
|
248
|
+
self.models = []
|
|
249
|
+
|
|
250
|
+
def fit(self, x):
|
|
251
|
+
self.ord_enc = []
|
|
252
|
+
self.ord_enc_joint = []
|
|
253
|
+
self.means = []
|
|
254
|
+
self.stds = []
|
|
255
|
+
for i in range(x.shape[1]):
|
|
256
|
+
d = x[:, i].clone()
|
|
257
|
+
df = pd.DataFrame(d, columns=["x"]).dropna()
|
|
258
|
+
model = DistTreeRegressor(max_depth=self.max_depth)
|
|
259
|
+
model.fit(df["x"].to_frame(), df["x"])
|
|
260
|
+
self.models.append(model)
|
|
261
|
+
preds = pd.DataFrame(model.predict_params(df["x"].to_frame()), columns=["mu", "sigma"])
|
|
262
|
+
ord_enc = OrdinalEncoder(dtype=int)
|
|
263
|
+
df_enc = ord_enc.fit_transform(preds)
|
|
264
|
+
self.ord_enc.append(ord_enc)
|
|
265
|
+
joint_group = np.char.add(np.char.add(df_enc[:, 0].astype(str), "_"), df_enc[:, 1].astype(str))
|
|
266
|
+
ord_enc_joint = OrdinalEncoder(dtype=int)
|
|
267
|
+
joint_group = ord_enc_joint.fit_transform(joint_group.reshape(-1, 1))
|
|
268
|
+
self.ord_enc_joint.append(ord_enc_joint)
|
|
269
|
+
df_aux = pd.DataFrame(
|
|
270
|
+
np.column_stack((preds.to_numpy(), joint_group)),
|
|
271
|
+
columns=["mu", "sigma", "group"],
|
|
272
|
+
)
|
|
273
|
+
self.means.append(df_aux.drop_duplicates(["group"]).sort_values("group")["mu"].to_list())
|
|
274
|
+
self.stds.append(df_aux.drop_duplicates(["group"]).sort_values("group")["sigma"].to_list())
|
|
275
|
+
|
|
276
|
+
def get_groups(self, x):
|
|
277
|
+
groups = []
|
|
278
|
+
for i in range(x.shape[1]):
|
|
279
|
+
d = x[:, i].clone()
|
|
280
|
+
miss_mask = d.isnan()
|
|
281
|
+
d[miss_mask] = d.nanmean()
|
|
282
|
+
preds = pd.DataFrame(self.models[i].predict_params(d.reshape(-1, 1)), columns=["mu", "sigma"])
|
|
283
|
+
df_enc = self.ord_enc[i].transform(preds)
|
|
284
|
+
joint_group = np.char.add(np.char.add(df_enc[:, 0].astype(str), "_"), df_enc[:, 1].astype(str))
|
|
285
|
+
joint_group = self.ord_enc_joint[i].transform(joint_group.reshape(-1, 1))
|
|
286
|
+
joint_group = joint_group.astype(float).flatten()
|
|
287
|
+
joint_group[miss_mask] = np.nan
|
|
288
|
+
groups.append(joint_group)
|
|
289
|
+
return np.column_stack(groups)
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import torch
|
|
4
|
+
from sklearn.mixture import BayesianGaussianMixture
|
|
5
|
+
from sklearn.preprocessing import OrdinalEncoder
|
|
6
|
+
|
|
7
|
+
from .disttree import DistTree
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Discretizer:
|
|
11
|
+
def __init__(
|
|
12
|
+
self,
|
|
13
|
+
X_num_trn,
|
|
14
|
+
variant="dt",
|
|
15
|
+
k_max=20,
|
|
16
|
+
perc_obs=0.03,
|
|
17
|
+
seed=42,
|
|
18
|
+
adjust_means=False,
|
|
19
|
+
max_depth=3,
|
|
20
|
+
):
|
|
21
|
+
self.seed = seed
|
|
22
|
+
self.variant = variant
|
|
23
|
+
self.adjust_means = adjust_means
|
|
24
|
+
self.max_depth = max_depth
|
|
25
|
+
self.k_max = k_max
|
|
26
|
+
self.perc_obs = perc_obs
|
|
27
|
+
self.fit_gmm_ord_enc = True
|
|
28
|
+
self.has_missing = torch.isnan(X_num_trn).any(0).numpy()
|
|
29
|
+
miss_mean = torch.nanmean(X_num_trn, dim=0)
|
|
30
|
+
|
|
31
|
+
if self.variant == "gmm":
|
|
32
|
+
self.gmms = self._train_bgmms(X_num_trn, k_max=k_max)
|
|
33
|
+
groups = self._get_gmm_groups(X_num_trn)
|
|
34
|
+
elif self.variant == "dt":
|
|
35
|
+
self.disttree = DistTree(max_depth)
|
|
36
|
+
self.disttree.fit(X_num_trn)
|
|
37
|
+
groups = self.disttree.get_groups(X_num_trn)
|
|
38
|
+
|
|
39
|
+
if self.adjust_means:
|
|
40
|
+
means, stds = [], []
|
|
41
|
+
for i in range(X_num_trn.shape[1]):
|
|
42
|
+
df = pd.DataFrame({"x": X_num_trn[:, i].clone(), "group": groups[:, i]})
|
|
43
|
+
df_stats = df.groupby("group").agg(["mean", "std"]).droplevel(0, axis=1)
|
|
44
|
+
means.append(torch.tensor(df_stats["mean"].to_numpy(), dtype=torch.float32))
|
|
45
|
+
stds.append(torch.tensor(df_stats["std"].to_numpy(), dtype=torch.float32))
|
|
46
|
+
elif self.variant == "gmm":
|
|
47
|
+
means = [torch.tensor(gmm.means_.squeeze(), dtype=torch.float32) for gmm in self.gmms]
|
|
48
|
+
stds = [torch.tensor(np.sqrt(gmm.covariances_.squeeze()), dtype=torch.float32) for gmm in self.gmms]
|
|
49
|
+
elif self.variant == "dt":
|
|
50
|
+
means = [torch.tensor(m, dtype=torch.float32) for m in self.disttree.means]
|
|
51
|
+
stds = [torch.tensor(s, dtype=torch.float32) for s in self.disttree.stds]
|
|
52
|
+
|
|
53
|
+
self.has_inflated = []
|
|
54
|
+
self.infl_groups = []
|
|
55
|
+
for i in range(X_num_trn.shape[1]):
|
|
56
|
+
df = pd.DataFrame({"x": X_num_trn[:, i].clone(), "group": groups[:, i]})
|
|
57
|
+
df_stats = df.groupby("group").agg(["mean", "std"]).droplevel(0, axis=1)
|
|
58
|
+
infl_idx = df_stats.loc[df_stats["std"] == 0].index.to_list()
|
|
59
|
+
self.has_inflated.append(len(infl_idx) > 0)
|
|
60
|
+
self.infl_groups.append(infl_idx)
|
|
61
|
+
stds[i][infl_idx] = 0
|
|
62
|
+
|
|
63
|
+
if self.has_missing.any():
|
|
64
|
+
for i in range(X_num_trn.shape[1]):
|
|
65
|
+
if self.has_missing[i]:
|
|
66
|
+
means[i] = torch.cat((miss_mean[i].unsqueeze(0), means[i]))
|
|
67
|
+
stds[i] = torch.cat((torch.zeros(1), stds[i]))
|
|
68
|
+
self.means = means
|
|
69
|
+
self.stds = stds
|
|
70
|
+
|
|
71
|
+
def encode(self, X):
|
|
72
|
+
groups = self._get_gmm_groups(X) if self.variant == "gmm" else self.disttree.get_groups(X)
|
|
73
|
+
return (*self.postprocess_groups(groups), self.infl_groups, self.has_missing)
|
|
74
|
+
|
|
75
|
+
def postprocess_groups(self, groups):
|
|
76
|
+
infl_mask = []
|
|
77
|
+
for i in range(groups.shape[1]):
|
|
78
|
+
infl_mask.append(torch.tensor(np.isin(groups[:, i], self.infl_groups[i]), dtype=torch.bool))
|
|
79
|
+
infl_mask = torch.column_stack(infl_mask)
|
|
80
|
+
|
|
81
|
+
miss_mask = []
|
|
82
|
+
for i in range(groups.shape[1]):
|
|
83
|
+
g_i = groups[:, i]
|
|
84
|
+
miss_mask.append(np.isnan(g_i))
|
|
85
|
+
if self.has_missing[i]:
|
|
86
|
+
groups[:, i] = np.nan_to_num(g_i.copy() + 1, nan=0, copy=True)
|
|
87
|
+
miss_mask = np.column_stack(miss_mask) if len(miss_mask) > 0 else None
|
|
88
|
+
miss_mask = torch.tensor(miss_mask, dtype=torch.bool) if self.has_missing.any() else None
|
|
89
|
+
return torch.tensor(groups, dtype=torch.long), miss_mask | infl_mask if miss_mask is not None else infl_mask
|
|
90
|
+
|
|
91
|
+
def _get_gmm_groups(self, X):
|
|
92
|
+
groups = []
|
|
93
|
+
for i in range(X.shape[1]):
|
|
94
|
+
d = X[:, i].clone()
|
|
95
|
+
miss_mask = d.isnan()
|
|
96
|
+
d[miss_mask] = d.nanmean()
|
|
97
|
+
group = self.gmms[i].predict(d.reshape(-1, 1)).astype(float)
|
|
98
|
+
group[miss_mask] = np.nan
|
|
99
|
+
groups.append(group)
|
|
100
|
+
groups = np.column_stack(groups)
|
|
101
|
+
if self.fit_gmm_ord_enc:
|
|
102
|
+
self.fit_gmm_ord_enc = False
|
|
103
|
+
self.gmm_ord_enc = OrdinalEncoder()
|
|
104
|
+
self.gmm_ord_enc.fit(groups)
|
|
105
|
+
return self.gmm_ord_enc.transform(groups)
|
|
106
|
+
|
|
107
|
+
def _train_bgmms(self, X, k_max=20):
|
|
108
|
+
bgmms = []
|
|
109
|
+
for i in range(X.shape[1]):
|
|
110
|
+
d = X[:, i].clone()
|
|
111
|
+
d = d[~d.isnan()]
|
|
112
|
+
bgmm = BayesianGaussianMixture(
|
|
113
|
+
n_components=k_max,
|
|
114
|
+
random_state=self.seed,
|
|
115
|
+
weight_concentration_prior_type="dirichlet_process",
|
|
116
|
+
weight_concentration_prior=0.001,
|
|
117
|
+
n_init=1,
|
|
118
|
+
)
|
|
119
|
+
bgmms.append(bgmm.fit(d.reshape(-1, 1)))
|
|
120
|
+
return bgmms
|