synthschema 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synthschema/__init__.py +56 -0
- synthschema/_version.py +1 -0
- synthschema/cli.py +134 -0
- synthschema/constraints.py +438 -0
- synthschema/copula.py +79 -0
- synthschema/datasets.py +106 -0
- synthschema/evaluation.py +446 -0
- synthschema/generators.py +456 -0
- synthschema/marginals.py +143 -0
- synthschema/metadata.py +547 -0
- synthschema/multi_table.py +272 -0
- synthschema/py.typed +0 -0
- synthschema/schema.py +490 -0
- synthschema/single_table.py +311 -0
- synthschema/transformers.py +247 -0
- synthschema-0.1.0.dist-info/METADATA +253 -0
- synthschema-0.1.0.dist-info/RECORD +20 -0
- synthschema-0.1.0.dist-info/WHEEL +4 -0
- synthschema-0.1.0.dist-info/entry_points.txt +2 -0
- synthschema-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
"""Single-table synthesis with a Gaussian copula."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import pickle
|
|
6
|
+
from collections.abc import Mapping, Sequence
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pandas as pd
|
|
12
|
+
from scipy import stats
|
|
13
|
+
|
|
14
|
+
from ._version import __version__
|
|
15
|
+
from .constraints import Constraint, build_constraints
|
|
16
|
+
from .copula import GaussianCopula
|
|
17
|
+
from .generators import IdGenerator, PiiGenerator, TextGenerator
|
|
18
|
+
from .marginals import EPS
|
|
19
|
+
from .metadata import Column, Metadata, Table, detect_column
|
|
20
|
+
from .transformers import make_handler, restore_dtype
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class GaussianCopulaSynthesizer:
|
|
24
|
+
"""Learns each column's distribution and the correlations between columns, then samples
|
|
25
|
+
new rows that keep that statistical balance.
|
|
26
|
+
|
|
27
|
+
* Numerical and datetime columns: exact empirical marginals by default (or a fitted
|
|
28
|
+
parametric family: ``norm``, ``beta``, ``truncnorm``, ``gamma``, ``uniform`` or ``auto``).
|
|
29
|
+
* Categorical and boolean columns: frequency-interval encoding, so category shares are kept.
|
|
30
|
+
* Correlations: a Gaussian copula over normal scores of every modelled column.
|
|
31
|
+
* Keys, PII and free text are generated, never learned, so real values are not copied.
|
|
32
|
+
* Constraints hold on 100% of rows.
|
|
33
|
+
|
|
34
|
+
Example::
|
|
35
|
+
|
|
36
|
+
synth = GaussianCopulaSynthesizer()
|
|
37
|
+
synth.fit(df)
|
|
38
|
+
fake = synth.sample(1_000, seed=42)
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(
|
|
42
|
+
self,
|
|
43
|
+
metadata: Metadata | None = None,
|
|
44
|
+
table_name: str | None = None,
|
|
45
|
+
*,
|
|
46
|
+
constraints: Sequence[Constraint | Mapping | str] | None = None,
|
|
47
|
+
default_distribution: str = "empirical",
|
|
48
|
+
numerical_distributions: Mapping[str, str] | None = None,
|
|
49
|
+
enforce_min_max_values: bool = True,
|
|
50
|
+
enforce_rounding: bool = True,
|
|
51
|
+
locale: str | list[str] | None = None,
|
|
52
|
+
) -> None:
|
|
53
|
+
self.metadata = metadata
|
|
54
|
+
self.table_name = table_name
|
|
55
|
+
self._user_constraints = list(constraints or [])
|
|
56
|
+
self.default_distribution = default_distribution
|
|
57
|
+
self.numerical_distributions = dict(numerical_distributions or {})
|
|
58
|
+
self.enforce_min_max_values = enforce_min_max_values
|
|
59
|
+
self.enforce_rounding = enforce_rounding
|
|
60
|
+
self.locale = locale
|
|
61
|
+
self._fitted = False
|
|
62
|
+
|
|
63
|
+
# ------------------------------------------------------------------ fit
|
|
64
|
+
def fit(self, data: pd.DataFrame) -> GaussianCopulaSynthesizer:
|
|
65
|
+
if not isinstance(data, pd.DataFrame):
|
|
66
|
+
raise TypeError("fit() expects a pandas DataFrame")
|
|
67
|
+
if len(data) == 0:
|
|
68
|
+
raise ValueError("Cannot fit on an empty DataFrame")
|
|
69
|
+
data = data.reset_index(drop=True)
|
|
70
|
+
if self.metadata is None:
|
|
71
|
+
self.metadata = Metadata.detect_from_dataframe(data, self.table_name or "table")
|
|
72
|
+
table = self.metadata.get_table(self.table_name)
|
|
73
|
+
self.table_name = table.name
|
|
74
|
+
missing = [c for c in table.columns if c not in data.columns]
|
|
75
|
+
if missing:
|
|
76
|
+
raise ValueError(f"Columns in metadata but not in data: {missing}")
|
|
77
|
+
|
|
78
|
+
columns = {str(c): table.columns.get(str(c)) for c in data.columns}
|
|
79
|
+
for name, meta in columns.items():
|
|
80
|
+
if meta is None:
|
|
81
|
+
columns[name] = detect_column(data[name], name)
|
|
82
|
+
self._table = Table(name=table.name, columns=columns, primary_key=table.primary_key)
|
|
83
|
+
self._columns = list(data.columns)
|
|
84
|
+
self._dtypes = dict(data.dtypes)
|
|
85
|
+
self._n_rows = len(data)
|
|
86
|
+
|
|
87
|
+
self._constraints = build_constraints(table.constraints) + build_constraints(self._user_constraints)
|
|
88
|
+
work = data
|
|
89
|
+
derived: dict[str, Column] = {}
|
|
90
|
+
for constraint in self._constraints:
|
|
91
|
+
constraint.fit(work, self._table)
|
|
92
|
+
work = constraint.transform(work)
|
|
93
|
+
derived.update(constraint.derived_columns())
|
|
94
|
+
|
|
95
|
+
rng = np.random.default_rng(0)
|
|
96
|
+
self._handlers: dict[str, Any] = {}
|
|
97
|
+
self._slices: dict[str, tuple[int, int]] = {}
|
|
98
|
+
self._generators: dict[str, Any] = {}
|
|
99
|
+
blocks = []
|
|
100
|
+
offset = 0
|
|
101
|
+
for col in work.columns:
|
|
102
|
+
meta = columns.get(col) or derived.get(col) or detect_column(work[col], col)
|
|
103
|
+
if meta.is_modeled:
|
|
104
|
+
dist = self.numerical_distributions.get(col, self.default_distribution)
|
|
105
|
+
handler = make_handler(meta, dist, self.enforce_min_max_values, self.enforce_rounding)
|
|
106
|
+
handler.fit(work[col])
|
|
107
|
+
block = handler.to_uniform(work[col], rng)
|
|
108
|
+
self._handlers[col] = handler
|
|
109
|
+
self._slices[col] = (offset, offset + block.shape[1])
|
|
110
|
+
offset += block.shape[1]
|
|
111
|
+
blocks.append(block)
|
|
112
|
+
elif meta.sdtype == "id":
|
|
113
|
+
self._generators[col] = IdGenerator(meta).fit(work[col])
|
|
114
|
+
elif meta.sdtype == "text":
|
|
115
|
+
self._generators[col] = TextGenerator(self.locale).fit(work[col])
|
|
116
|
+
else:
|
|
117
|
+
self._generators[col] = PiiGenerator(meta.sdtype, self.locale).fit(work[col])
|
|
118
|
+
|
|
119
|
+
U = np.hstack(blocks) if blocks else np.zeros((len(work), 0))
|
|
120
|
+
Z = stats.norm.ppf(np.clip(U, EPS, 1 - EPS))
|
|
121
|
+
self._copula = GaussianCopula().fit(Z)
|
|
122
|
+
self._fitted = True
|
|
123
|
+
return self
|
|
124
|
+
|
|
125
|
+
# ------------------------------------------------------------------ sampling internals
|
|
126
|
+
def _check_fitted(self) -> None:
|
|
127
|
+
if not self._fitted:
|
|
128
|
+
raise RuntimeError("Call fit() before sampling")
|
|
129
|
+
|
|
130
|
+
def _latent_given(self, cond: pd.DataFrame) -> tuple[list[int], np.ndarray]:
|
|
131
|
+
idx: list[int] = []
|
|
132
|
+
values = []
|
|
133
|
+
for col in cond.columns:
|
|
134
|
+
start, stop = self._slices[col]
|
|
135
|
+
U = self._handlers[col].to_uniform(cond[col].reset_index(drop=True), None)
|
|
136
|
+
idx.extend(range(start, stop))
|
|
137
|
+
values.append(stats.norm.ppf(U))
|
|
138
|
+
return idx, np.hstack(values)
|
|
139
|
+
|
|
140
|
+
def _decode(self, Z: np.ndarray, cond: pd.DataFrame | None = None) -> pd.DataFrame:
|
|
141
|
+
U = np.clip(stats.norm.cdf(Z), EPS, 1 - EPS)
|
|
142
|
+
out = pd.DataFrame(index=pd.RangeIndex(len(Z)))
|
|
143
|
+
for col, handler in self._handlers.items():
|
|
144
|
+
start, stop = self._slices[col]
|
|
145
|
+
out[col] = handler.from_uniform(U[:, start:stop]).to_numpy()
|
|
146
|
+
if cond is not None:
|
|
147
|
+
for col in cond.columns: # conditioned values are kept exactly
|
|
148
|
+
out[col] = cond[col].to_numpy()
|
|
149
|
+
for constraint in reversed(self._constraints):
|
|
150
|
+
out = constraint.reverse(out)
|
|
151
|
+
return out
|
|
152
|
+
|
|
153
|
+
def _sample_modeled(self, n: int, rng: np.random.Generator, cond: pd.DataFrame | None) -> pd.DataFrame:
|
|
154
|
+
if cond is not None and len(cond.columns):
|
|
155
|
+
idx, given = self._latent_given(cond)
|
|
156
|
+
Z = self._copula.sample_conditional(idx, given, rng)
|
|
157
|
+
else:
|
|
158
|
+
Z = self._copula.sample(n, rng)
|
|
159
|
+
cond = None
|
|
160
|
+
return self._decode(Z, cond)
|
|
161
|
+
|
|
162
|
+
def _valid(self, df: pd.DataFrame) -> np.ndarray:
|
|
163
|
+
mask = np.ones(len(df), dtype=bool)
|
|
164
|
+
for constraint in self._constraints:
|
|
165
|
+
mask &= constraint.is_valid(df)
|
|
166
|
+
return mask
|
|
167
|
+
|
|
168
|
+
def _generate(
|
|
169
|
+
self,
|
|
170
|
+
n: int,
|
|
171
|
+
rng: np.random.Generator,
|
|
172
|
+
cond: pd.DataFrame | None = None,
|
|
173
|
+
on_fail: str = "raise",
|
|
174
|
+
max_tries: int = 100,
|
|
175
|
+
) -> pd.DataFrame:
|
|
176
|
+
"""Sample ``n`` complete rows. With ``cond``, row ``i`` is conditioned on ``cond.iloc[i]``.
|
|
177
|
+
The returned index is the position of each row's condition (rows may be dropped only
|
|
178
|
+
when ``on_fail="drop"``)."""
|
|
179
|
+
self._check_fitted()
|
|
180
|
+
if cond is not None:
|
|
181
|
+
cond = cond.reset_index(drop=True)
|
|
182
|
+
df = self._sample_modeled(n, rng, cond)
|
|
183
|
+
valid = self._valid(df)
|
|
184
|
+
tries = 0
|
|
185
|
+
while not valid.all() and tries < max_tries:
|
|
186
|
+
bad = np.flatnonzero(~valid)
|
|
187
|
+
redo = self._sample_modeled(len(bad), rng, None if cond is None else cond.iloc[bad])
|
|
188
|
+
for col in redo.columns:
|
|
189
|
+
if df[col].dtype != redo[col].dtype:
|
|
190
|
+
df[col] = df[col].astype(object)
|
|
191
|
+
df.loc[bad, col] = redo[col].to_numpy()
|
|
192
|
+
valid[bad] = self._valid(redo)
|
|
193
|
+
tries += 1
|
|
194
|
+
if not valid.all():
|
|
195
|
+
if on_fail == "raise":
|
|
196
|
+
raise ValueError(
|
|
197
|
+
f"Could not satisfy the constraints for {(~valid).sum()} of {n} rows after "
|
|
198
|
+
f"{max_tries} attempts; check that the rules agree with each other and the data"
|
|
199
|
+
)
|
|
200
|
+
df = df[valid]
|
|
201
|
+
m = len(df)
|
|
202
|
+
pk = self._table.primary_key
|
|
203
|
+
for col, gen in self._generators.items():
|
|
204
|
+
if col == pk:
|
|
205
|
+
df[col] = gen.generate(m, rng, unique=True, allow_nulls=False)
|
|
206
|
+
else:
|
|
207
|
+
df[col] = gen.generate(m, rng)
|
|
208
|
+
df = df[self._columns]
|
|
209
|
+
for col in self._columns:
|
|
210
|
+
df[col] = restore_dtype(df[col], self._dtypes[col])
|
|
211
|
+
return df
|
|
212
|
+
|
|
213
|
+
# ------------------------------------------------------------------ public sampling
|
|
214
|
+
def sample(
|
|
215
|
+
self,
|
|
216
|
+
num_rows: int,
|
|
217
|
+
conditions: Mapping[str, Any] | None = None,
|
|
218
|
+
seed: int | None = None,
|
|
219
|
+
max_tries: int = 100,
|
|
220
|
+
) -> pd.DataFrame:
|
|
221
|
+
"""Generate ``num_rows`` synthetic rows.
|
|
222
|
+
|
|
223
|
+
``conditions`` fixes column values for every row, e.g. ``{"region": "North"}``. The
|
|
224
|
+
other columns are drawn from the copula *conditioned on* those values, so their
|
|
225
|
+
correlations with the fixed columns are kept. ``seed`` makes the output reproducible.
|
|
226
|
+
"""
|
|
227
|
+
self._check_fitted()
|
|
228
|
+
if num_rows < 0:
|
|
229
|
+
raise ValueError("num_rows must be >= 0")
|
|
230
|
+
rng = np.random.default_rng(seed)
|
|
231
|
+
if not conditions:
|
|
232
|
+
return self._generate(num_rows, rng, max_tries=max_tries).reset_index(drop=True)
|
|
233
|
+
unknown = [c for c in conditions if c not in self._columns]
|
|
234
|
+
if unknown:
|
|
235
|
+
raise KeyError(f"Unknown condition column(s): {unknown}")
|
|
236
|
+
direct = {k: v for k, v in conditions.items() if k in self._handlers}
|
|
237
|
+
indirect = {k: v for k, v in conditions.items() if k not in self._handlers}
|
|
238
|
+
cond = pd.DataFrame({k: [v] * num_rows for k, v in direct.items()}) if direct else None
|
|
239
|
+
if not indirect:
|
|
240
|
+
return self._generate(num_rows, rng, cond, max_tries=max_tries).reset_index(drop=True)
|
|
241
|
+
return self._reject_sample(num_rows, rng, direct, indirect, max_tries)
|
|
242
|
+
|
|
243
|
+
def _reject_sample(self, num_rows, rng, direct, indirect, max_tries) -> pd.DataFrame:
|
|
244
|
+
batch = max(num_rows * 4, 200)
|
|
245
|
+
cond = pd.DataFrame({k: [v] * batch for k, v in direct.items()}) if direct else None
|
|
246
|
+
kept: list[pd.DataFrame] = []
|
|
247
|
+
total = 0
|
|
248
|
+
for _ in range(max_tries):
|
|
249
|
+
df = self._generate(batch, rng, cond, on_fail="drop", max_tries=max_tries)
|
|
250
|
+
mask = np.ones(len(df), dtype=bool)
|
|
251
|
+
for col, value in indirect.items():
|
|
252
|
+
mask &= (df[col] == value).to_numpy()
|
|
253
|
+
if mask.any():
|
|
254
|
+
kept.append(df[mask])
|
|
255
|
+
total += int(mask.sum())
|
|
256
|
+
if total >= num_rows:
|
|
257
|
+
break
|
|
258
|
+
if total < num_rows:
|
|
259
|
+
raise ValueError(
|
|
260
|
+
f"Only {total} of {num_rows} rows matched the conditions {indirect}; these "
|
|
261
|
+
"columns are shaped by constraints, so they are matched by rejection sampling"
|
|
262
|
+
)
|
|
263
|
+
return pd.concat(kept).head(num_rows).reset_index(drop=True)
|
|
264
|
+
|
|
265
|
+
def sample_from_conditions(self, conditions: pd.DataFrame, seed: int | None = None) -> pd.DataFrame:
|
|
266
|
+
"""One synthetic row per row of ``conditions``; non-null cells are fixed values."""
|
|
267
|
+
self._check_fitted()
|
|
268
|
+
rng = np.random.default_rng(seed)
|
|
269
|
+
unknown = [c for c in conditions.columns if c not in self._handlers]
|
|
270
|
+
if unknown:
|
|
271
|
+
raise KeyError(f"sample_from_conditions supports modelled columns only; got {unknown}")
|
|
272
|
+
conditions = conditions.reset_index(drop=True)
|
|
273
|
+
mask = conditions.notna()
|
|
274
|
+
pieces = []
|
|
275
|
+
for pattern, group in conditions.groupby(mask.apply(tuple, axis=1), sort=False):
|
|
276
|
+
cols = [c for c, present in zip(conditions.columns, pattern, strict=False) if present]
|
|
277
|
+
cond = group[cols] if cols else None
|
|
278
|
+
out = self._generate(len(group), rng, cond)
|
|
279
|
+
out.index = group.index
|
|
280
|
+
pieces.append(out)
|
|
281
|
+
return pd.concat(pieces).sort_index()
|
|
282
|
+
|
|
283
|
+
# ------------------------------------------------------------------ inspection and io
|
|
284
|
+
def get_metadata(self) -> Metadata:
|
|
285
|
+
return self.metadata
|
|
286
|
+
|
|
287
|
+
def get_learned_correlations(self) -> pd.DataFrame:
|
|
288
|
+
"""Correlation of normal scores between modelled (latent) columns."""
|
|
289
|
+
self._check_fitted()
|
|
290
|
+
names = []
|
|
291
|
+
for col, (start, stop) in self._slices.items():
|
|
292
|
+
names.extend([col] if stop - start == 1 else [f"{col}.is_null", col])
|
|
293
|
+
return pd.DataFrame(self._copula.corr, index=names, columns=names)
|
|
294
|
+
|
|
295
|
+
def save(self, path: str | Path) -> None:
|
|
296
|
+
"""Save the fitted synthesizer (pickle). Only load files you trust."""
|
|
297
|
+
with open(path, "wb") as fh:
|
|
298
|
+
pickle.dump({"synthschema_version": __version__, "synthesizer": self}, fh)
|
|
299
|
+
|
|
300
|
+
@classmethod
|
|
301
|
+
def load(cls, path: str | Path) -> GaussianCopulaSynthesizer:
|
|
302
|
+
with open(path, "rb") as fh:
|
|
303
|
+
payload = pickle.load(fh) # noqa: S301 - documented: trusted files only
|
|
304
|
+
obj = payload["synthesizer"] if isinstance(payload, dict) else payload
|
|
305
|
+
if not isinstance(obj, cls):
|
|
306
|
+
raise TypeError(f"{path} does not contain a {cls.__name__}")
|
|
307
|
+
return obj
|
|
308
|
+
|
|
309
|
+
def __repr__(self) -> str:
|
|
310
|
+
state = f"fitted on {self._n_rows} rows" if self._fitted else "not fitted"
|
|
311
|
+
return f"GaussianCopulaSynthesizer(table={self.table_name!r}, {state})"
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""Reversible column transformers between data values and the copula's uniform space.
|
|
2
|
+
|
|
3
|
+
Each handler maps one column to one or more columns of values in (0, 1) and back.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
from .marginals import EPS, fit_marginal
|
|
12
|
+
from .metadata import Column
|
|
13
|
+
|
|
14
|
+
_NULL = "__synthschema_null__"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def learn_decimals(values: np.ndarray, max_decimals: int = 6) -> int | None:
|
|
18
|
+
values = values[np.isfinite(values)]
|
|
19
|
+
if len(values) == 0:
|
|
20
|
+
return None
|
|
21
|
+
sample = values if len(values) <= 10_000 else values[:: max(1, len(values) // 10_000)]
|
|
22
|
+
for d in range(max_decimals + 1):
|
|
23
|
+
scaled = sample * 10**d
|
|
24
|
+
if np.allclose(scaled, np.round(scaled), rtol=0, atol=1e-6 * max(1.0, 10**d)):
|
|
25
|
+
if d == 0 and len(np.unique(sample)) > 10:
|
|
26
|
+
# Whole numbers: also learn rounding to tens, hundreds, ... (e.g. salaries).
|
|
27
|
+
ints = np.round(sample).astype(np.int64)
|
|
28
|
+
for k in range(6, 0, -1):
|
|
29
|
+
if (ints % 10**k == 0).all():
|
|
30
|
+
return -k
|
|
31
|
+
return d
|
|
32
|
+
return None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class CategoricalHandler:
|
|
36
|
+
"""Frequency-interval encoding: each category owns a slice of (0, 1) sized by its share."""
|
|
37
|
+
|
|
38
|
+
n_latent = 1
|
|
39
|
+
|
|
40
|
+
def fit(self, series: pd.Series) -> CategoricalHandler:
|
|
41
|
+
values = series.astype(object).where(series.notna(), _NULL)
|
|
42
|
+
counts = values.value_counts(sort=True)
|
|
43
|
+
self.categories = list(counts.index)
|
|
44
|
+
p = counts.to_numpy(dtype=float)
|
|
45
|
+
p = p / p.sum()
|
|
46
|
+
self.upper = np.cumsum(p)
|
|
47
|
+
self.upper[-1] = 1.0
|
|
48
|
+
self.lower = np.concatenate([[0.0], self.upper[:-1]])
|
|
49
|
+
self.index = {c: i for i, c in enumerate(self.categories)}
|
|
50
|
+
self.probabilities = p
|
|
51
|
+
return self
|
|
52
|
+
|
|
53
|
+
def to_uniform(self, series: pd.Series, rng: np.random.Generator | None = None) -> np.ndarray:
|
|
54
|
+
values = series.astype(object).where(series.notna(), _NULL)
|
|
55
|
+
idx = values.map(self.index)
|
|
56
|
+
unknown = idx.isna().to_numpy()
|
|
57
|
+
idx = idx.fillna(0).astype(int).to_numpy()
|
|
58
|
+
lo, hi = self.lower[idx], self.upper[idx]
|
|
59
|
+
frac = rng.random(len(idx)) if rng is not None else 0.5
|
|
60
|
+
u = lo + frac * (hi - lo)
|
|
61
|
+
if unknown.any():
|
|
62
|
+
u[unknown] = rng.random(unknown.sum()) if rng is not None else 0.5
|
|
63
|
+
return np.clip(u, EPS, 1 - EPS)[:, None]
|
|
64
|
+
|
|
65
|
+
def from_uniform(self, U: np.ndarray) -> pd.Series:
|
|
66
|
+
idx = np.searchsorted(self.upper, U[:, 0], side="right")
|
|
67
|
+
idx = np.clip(idx, 0, len(self.categories) - 1)
|
|
68
|
+
out = np.array(self.categories, dtype=object)[idx]
|
|
69
|
+
out[out == _NULL] = None
|
|
70
|
+
return pd.Series(out, dtype=object)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class NumericalHandler:
|
|
74
|
+
"""A fitted marginal plus an optional null indicator, min/max clipping and rounding."""
|
|
75
|
+
|
|
76
|
+
def __init__(
|
|
77
|
+
self,
|
|
78
|
+
distribution: str = "empirical",
|
|
79
|
+
enforce_min_max: bool = True,
|
|
80
|
+
enforce_rounding: bool = True,
|
|
81
|
+
) -> None:
|
|
82
|
+
self.distribution = distribution
|
|
83
|
+
self.enforce_min_max = enforce_min_max
|
|
84
|
+
self.enforce_rounding = enforce_rounding
|
|
85
|
+
|
|
86
|
+
def _values(self, series: pd.Series) -> np.ndarray:
|
|
87
|
+
return pd.to_numeric(series, errors="coerce").astype(float).to_numpy()
|
|
88
|
+
|
|
89
|
+
def fit(self, series: pd.Series) -> NumericalHandler:
|
|
90
|
+
x = self._values(series)
|
|
91
|
+
nulls = np.isnan(x)
|
|
92
|
+
self.null_rate = float(nulls.mean()) if len(x) else 0.0
|
|
93
|
+
self.has_nulls = 0 < self.null_rate < 1
|
|
94
|
+
self.all_null = len(x) == 0 or self.null_rate == 1
|
|
95
|
+
vals = x[~nulls]
|
|
96
|
+
self.min = float(vals.min()) if len(vals) else 0.0
|
|
97
|
+
self.max = float(vals.max()) if len(vals) else 0.0
|
|
98
|
+
self.decimals = learn_decimals(vals) if self.enforce_rounding else None
|
|
99
|
+
self.marginal = fit_marginal(vals, self.distribution)
|
|
100
|
+
self.null_handler = CategoricalHandler().fit(pd.Series(nulls)) if self.has_nulls else None
|
|
101
|
+
return self
|
|
102
|
+
|
|
103
|
+
@property
|
|
104
|
+
def n_latent(self) -> int:
|
|
105
|
+
return 2 if self.has_nulls else 1
|
|
106
|
+
|
|
107
|
+
def to_uniform(self, series: pd.Series, rng: np.random.Generator | None = None) -> np.ndarray:
|
|
108
|
+
x = self._values(series)
|
|
109
|
+
nulls = np.isnan(x)
|
|
110
|
+
u = np.full(len(x), 0.5)
|
|
111
|
+
if (~nulls).any() and not self.all_null:
|
|
112
|
+
u[~nulls] = self.marginal.cdf(x[~nulls], rng)
|
|
113
|
+
if self.has_nulls:
|
|
114
|
+
return np.column_stack([self.null_handler.to_uniform(pd.Series(nulls), rng)[:, 0], u])
|
|
115
|
+
return u[:, None]
|
|
116
|
+
|
|
117
|
+
def from_uniform(self, U: np.ndarray) -> pd.Series:
|
|
118
|
+
if self.all_null:
|
|
119
|
+
return pd.Series(np.full(len(U), np.nan))
|
|
120
|
+
values = self.marginal.ppf(U[:, -1])
|
|
121
|
+
if self.enforce_min_max:
|
|
122
|
+
values = np.clip(values, self.min, self.max)
|
|
123
|
+
if self.decimals is not None:
|
|
124
|
+
values = np.round(values, self.decimals)
|
|
125
|
+
if self.has_nulls:
|
|
126
|
+
isnull = self.null_handler.from_uniform(U[:, :1]).astype(bool).to_numpy()
|
|
127
|
+
values = values.astype(float)
|
|
128
|
+
values[isnull] = np.nan
|
|
129
|
+
return pd.Series(values)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
_GRANULARITY_NS = {"day": 86_400 * 10**9, "second": 10**9}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def to_datetime(series: pd.Series, fmt: str | None) -> pd.Series:
|
|
136
|
+
if pd.api.types.is_datetime64_any_dtype(series):
|
|
137
|
+
out = series
|
|
138
|
+
elif fmt:
|
|
139
|
+
out = pd.to_datetime(series, format=fmt, errors="coerce")
|
|
140
|
+
else:
|
|
141
|
+
out = pd.to_datetime(series, format="ISO8601", errors="coerce")
|
|
142
|
+
if getattr(out.dt, "tz", None) is not None:
|
|
143
|
+
out = out.dt.tz_convert("UTC").dt.tz_localize(None)
|
|
144
|
+
return out
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def datetime_to_seconds(series: pd.Series, fmt: str | None) -> np.ndarray:
|
|
148
|
+
dt = to_datetime(series, fmt)
|
|
149
|
+
ns = dt.to_numpy(dtype="datetime64[ns]").astype("int64").astype(float)
|
|
150
|
+
ns[dt.isna().to_numpy()] = np.nan
|
|
151
|
+
return ns / 1e9
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def learn_granularity(seconds: np.ndarray) -> str | None:
|
|
155
|
+
vals = seconds[np.isfinite(seconds)]
|
|
156
|
+
if len(vals) == 0:
|
|
157
|
+
return None
|
|
158
|
+
ns = np.round(vals * 1e9).astype(np.int64)
|
|
159
|
+
if (ns % _GRANULARITY_NS["day"] == 0).all():
|
|
160
|
+
return "day"
|
|
161
|
+
if (ns % _GRANULARITY_NS["second"] == 0).all():
|
|
162
|
+
return "second"
|
|
163
|
+
return None
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def seconds_to_datetime(seconds: np.ndarray, granularity: str | None) -> pd.Series:
|
|
167
|
+
seconds = np.asarray(seconds, dtype=float)
|
|
168
|
+
out = pd.to_datetime(pd.Series(seconds), unit="s", errors="coerce")
|
|
169
|
+
if granularity == "day":
|
|
170
|
+
out = out.dt.floor("D")
|
|
171
|
+
elif granularity == "second":
|
|
172
|
+
out = out.dt.round("s")
|
|
173
|
+
return out
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def format_datetime(dt: pd.Series, fmt: str | None, as_string: bool) -> pd.Series:
|
|
177
|
+
if not as_string:
|
|
178
|
+
return dt
|
|
179
|
+
if fmt:
|
|
180
|
+
out = dt.dt.strftime(fmt)
|
|
181
|
+
else:
|
|
182
|
+
out = dt.map(lambda v: v.isoformat() if pd.notna(v) else None)
|
|
183
|
+
return out.astype(object).where(dt.notna(), None)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class DatetimeHandler:
|
|
187
|
+
"""Datetimes are modelled as seconds since the epoch and restored in their own format."""
|
|
188
|
+
|
|
189
|
+
def __init__(self, column: Column | None = None, distribution: str = "empirical",
|
|
190
|
+
enforce_min_max: bool = True) -> None:
|
|
191
|
+
self.fmt = column.datetime_format if column else None
|
|
192
|
+
self.numeric = NumericalHandler(distribution, enforce_min_max, enforce_rounding=False)
|
|
193
|
+
|
|
194
|
+
def fit(self, series: pd.Series) -> DatetimeHandler:
|
|
195
|
+
self.as_string = not pd.api.types.is_datetime64_any_dtype(series)
|
|
196
|
+
seconds = datetime_to_seconds(series, self.fmt)
|
|
197
|
+
self.granularity = learn_granularity(seconds)
|
|
198
|
+
self.numeric.fit(pd.Series(seconds))
|
|
199
|
+
return self
|
|
200
|
+
|
|
201
|
+
@property
|
|
202
|
+
def n_latent(self) -> int:
|
|
203
|
+
return self.numeric.n_latent
|
|
204
|
+
|
|
205
|
+
def to_uniform(self, series: pd.Series, rng: np.random.Generator | None = None) -> np.ndarray:
|
|
206
|
+
return self.numeric.to_uniform(pd.Series(datetime_to_seconds(series, self.fmt)), rng)
|
|
207
|
+
|
|
208
|
+
def from_uniform(self, U: np.ndarray) -> pd.Series:
|
|
209
|
+
seconds = self.numeric.from_uniform(U).to_numpy()
|
|
210
|
+
dt = seconds_to_datetime(seconds, self.granularity)
|
|
211
|
+
return format_datetime(dt, self.fmt, self.as_string)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def make_handler(column: Column, distribution: str = "empirical", enforce_min_max: bool = True,
|
|
215
|
+
enforce_rounding: bool = True):
|
|
216
|
+
if column.sdtype == "numerical":
|
|
217
|
+
return NumericalHandler(distribution, enforce_min_max, enforce_rounding)
|
|
218
|
+
if column.sdtype == "datetime":
|
|
219
|
+
return DatetimeHandler(column, distribution, enforce_min_max)
|
|
220
|
+
if column.sdtype in {"categorical", "boolean"}:
|
|
221
|
+
return CategoricalHandler()
|
|
222
|
+
raise ValueError(f"sdtype {column.sdtype!r} is not modelled")
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def restore_dtype(series: pd.Series, dtype) -> pd.Series:
|
|
226
|
+
"""Cast a generated column back to the dtype of the training column, where possible."""
|
|
227
|
+
try:
|
|
228
|
+
if isinstance(dtype, pd.CategoricalDtype):
|
|
229
|
+
return series.astype(dtype)
|
|
230
|
+
if pd.api.types.is_bool_dtype(dtype):
|
|
231
|
+
if series.notna().all():
|
|
232
|
+
return series.astype(bool)
|
|
233
|
+
return series.astype("boolean")
|
|
234
|
+
if pd.api.types.is_integer_dtype(dtype):
|
|
235
|
+
num = pd.to_numeric(series, errors="coerce").round()
|
|
236
|
+
if num.notna().all() and isinstance(dtype, np.dtype):
|
|
237
|
+
return num.astype(dtype)
|
|
238
|
+
return num.astype("Int64")
|
|
239
|
+
if pd.api.types.is_float_dtype(dtype):
|
|
240
|
+
return pd.to_numeric(series, errors="coerce").astype(dtype)
|
|
241
|
+
if pd.api.types.is_datetime64_any_dtype(dtype):
|
|
242
|
+
return pd.to_datetime(series, errors="coerce")
|
|
243
|
+
if isinstance(dtype, pd.StringDtype):
|
|
244
|
+
return series.astype(dtype)
|
|
245
|
+
except (TypeError, ValueError):
|
|
246
|
+
return series
|
|
247
|
+
return series
|