synthschema 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synthschema/__init__.py +56 -0
- synthschema/_version.py +1 -0
- synthschema/cli.py +134 -0
- synthschema/constraints.py +438 -0
- synthschema/copula.py +79 -0
- synthschema/datasets.py +106 -0
- synthschema/evaluation.py +446 -0
- synthschema/generators.py +456 -0
- synthschema/marginals.py +143 -0
- synthschema/metadata.py +547 -0
- synthschema/multi_table.py +272 -0
- synthschema/py.typed +0 -0
- synthschema/schema.py +490 -0
- synthschema/single_table.py +311 -0
- synthschema/transformers.py +247 -0
- synthschema-0.1.0.dist-info/METADATA +253 -0
- synthschema-0.1.0.dist-info/RECORD +20 -0
- synthschema-0.1.0.dist-info/WHEEL +4 -0
- synthschema-0.1.0.dist-info/entry_points.txt +2 -0
- synthschema-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
"""Multi-table synthesis: parent-first, conditional generation that keeps keys intact.
|
|
2
|
+
|
|
3
|
+
Tables are generated in topological order. For each relationship the parent's model also
|
|
4
|
+
learns *how many children* each parent row has (correlated with the parent's attributes),
|
|
5
|
+
and the child's model learns its columns *conditioned on the parent row*. This:
|
|
6
|
+
|
|
7
|
+
* keeps referential integrity by construction (no orphan foreign keys),
|
|
8
|
+
* keeps the child-count distribution per parent,
|
|
9
|
+
* keeps cross-table patterns (for example, high-income customers placing larger orders),
|
|
10
|
+
* scales linearly with the number of tables.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import pickle
|
|
16
|
+
from collections.abc import Mapping
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
import pandas as pd
|
|
22
|
+
|
|
23
|
+
from ._version import __version__
|
|
24
|
+
from .generators import IdGenerator
|
|
25
|
+
from .metadata import Column, Metadata, Relationship, detect_column
|
|
26
|
+
from .single_table import GaussianCopulaSynthesizer
|
|
27
|
+
from .transformers import restore_dtype
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _count_col(rel: Relationship) -> str:
|
|
31
|
+
return f"__count__{rel.child_table}__{rel.child_foreign_key}"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _parent_col(col: str) -> str:
|
|
35
|
+
return f"__parent__{col}"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class _TableInfo:
|
|
40
|
+
n_rows: int
|
|
41
|
+
columns: list[str]
|
|
42
|
+
dtypes: dict
|
|
43
|
+
primary_key: str | None
|
|
44
|
+
pk_generator: IdGenerator | None
|
|
45
|
+
primary: Relationship | None
|
|
46
|
+
secondary: list[Relationship] = field(default_factory=list)
|
|
47
|
+
self_refs: list[Relationship] = field(default_factory=list)
|
|
48
|
+
context: list[str] = field(default_factory=list)
|
|
49
|
+
null_rates: dict[str, float] = field(default_factory=dict)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class MultiTableSynthesizer:
|
|
53
|
+
"""Synthesizes a set of related tables.
|
|
54
|
+
|
|
55
|
+
Example::
|
|
56
|
+
|
|
57
|
+
metadata = Metadata.detect_from_dataframes({"customers": customers, "orders": orders})
|
|
58
|
+
synth = MultiTableSynthesizer(metadata)
|
|
59
|
+
synth.fit({"customers": customers, "orders": orders})
|
|
60
|
+
fake = synth.sample(scale=1.0, seed=7) # dict of DataFrames
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
def __init__(
|
|
64
|
+
self,
|
|
65
|
+
metadata: Metadata,
|
|
66
|
+
*,
|
|
67
|
+
default_distribution: str = "empirical",
|
|
68
|
+
numerical_distributions: Mapping[str, Mapping[str, str]] | None = None,
|
|
69
|
+
locale: str | list[str] | None = None,
|
|
70
|
+
max_context_columns: int = 20,
|
|
71
|
+
) -> None:
|
|
72
|
+
self.metadata = metadata
|
|
73
|
+
self.default_distribution = default_distribution
|
|
74
|
+
self.numerical_distributions = dict(numerical_distributions or {})
|
|
75
|
+
self.locale = locale
|
|
76
|
+
self.max_context_columns = max_context_columns
|
|
77
|
+
self._fitted = False
|
|
78
|
+
|
|
79
|
+
def _keys(self, table: str) -> set[str]:
|
|
80
|
+
t = self.metadata.tables[table]
|
|
81
|
+
keys = {r.child_foreign_key for r in self.metadata.relationships if r.child_table == table}
|
|
82
|
+
if t.primary_key:
|
|
83
|
+
keys.add(t.primary_key)
|
|
84
|
+
return keys
|
|
85
|
+
|
|
86
|
+
def _context_columns(self, table: str) -> list[str]:
|
|
87
|
+
t = self.metadata.tables[table]
|
|
88
|
+
keys = self._keys(table)
|
|
89
|
+
cols = [c for c, meta in t.columns.items() if meta.is_modeled and c not in keys]
|
|
90
|
+
return cols[: self.max_context_columns]
|
|
91
|
+
|
|
92
|
+
# ------------------------------------------------------------------ fit
|
|
93
|
+
def fit(self, data: Mapping[str, pd.DataFrame]) -> MultiTableSynthesizer:
|
|
94
|
+
md = self.metadata
|
|
95
|
+
md.validate()
|
|
96
|
+
md.validate_data(data)
|
|
97
|
+
self._order = md.topological_order()
|
|
98
|
+
self._synths: dict[str, GaussianCopulaSynthesizer] = {}
|
|
99
|
+
self._info: dict[str, _TableInfo] = {}
|
|
100
|
+
|
|
101
|
+
for name in self._order:
|
|
102
|
+
table = md.tables[name]
|
|
103
|
+
df = data[name].reset_index(drop=True)
|
|
104
|
+
pk = table.primary_key
|
|
105
|
+
if pk is not None and (df[pk].isna().any() or not df[pk].is_unique):
|
|
106
|
+
raise ValueError(f"Primary key {name}.{pk} must be unique and non-null")
|
|
107
|
+
parents = md.parent_relationships(name)
|
|
108
|
+
children = md.child_relationships(name)
|
|
109
|
+
self_refs = [r for r in md.relationships if r.parent_table == r.child_table == name]
|
|
110
|
+
keys = self._keys(name)
|
|
111
|
+
|
|
112
|
+
own = [c for c in df.columns if c not in keys]
|
|
113
|
+
aug = df[own].copy()
|
|
114
|
+
aug_meta: dict[str, Column] = {
|
|
115
|
+
c: table.columns[c] if c in table.columns else detect_column(df[c], c) for c in own
|
|
116
|
+
}
|
|
117
|
+
for rel in children:
|
|
118
|
+
if pk is None:
|
|
119
|
+
raise ValueError(f"Table {name!r} has children but no primary key")
|
|
120
|
+
counts = data[rel.child_table][rel.child_foreign_key].value_counts()
|
|
121
|
+
aug[_count_col(rel)] = df[pk].map(counts).fillna(0).astype(int).to_numpy()
|
|
122
|
+
aug_meta[_count_col(rel)] = Column("numerical", representation="int")
|
|
123
|
+
|
|
124
|
+
primary = parents[0] if parents else None
|
|
125
|
+
null_rates: dict[str, float] = {}
|
|
126
|
+
context: list[str] = []
|
|
127
|
+
if primary is not None:
|
|
128
|
+
ptable = md.tables[primary.parent_table]
|
|
129
|
+
pdf = data[primary.parent_table]
|
|
130
|
+
context = self._context_columns(primary.parent_table)
|
|
131
|
+
lookup = pdf.set_index(ptable.primary_key)[context]
|
|
132
|
+
fk = df[primary.child_foreign_key]
|
|
133
|
+
joined = lookup.reindex(fk.to_numpy())
|
|
134
|
+
for c in context:
|
|
135
|
+
aug[_parent_col(c)] = joined[c].to_numpy()
|
|
136
|
+
aug_meta[_parent_col(c)] = ptable.columns[c]
|
|
137
|
+
null_rates[primary.child_foreign_key] = float(1 - fk.isin(pdf[ptable.primary_key]).mean())
|
|
138
|
+
for rel in parents[1:] + self_refs:
|
|
139
|
+
null_rates[rel.child_foreign_key] = float(df[rel.child_foreign_key].isna().mean())
|
|
140
|
+
|
|
141
|
+
sub = Metadata()
|
|
142
|
+
sub_table = sub.add_table(name)
|
|
143
|
+
sub_table.columns = aug_meta
|
|
144
|
+
sub_table.constraints = list(table.constraints)
|
|
145
|
+
synth = GaussianCopulaSynthesizer(
|
|
146
|
+
sub,
|
|
147
|
+
name,
|
|
148
|
+
default_distribution=self.default_distribution,
|
|
149
|
+
numerical_distributions=self.numerical_distributions.get(name),
|
|
150
|
+
locale=self.locale,
|
|
151
|
+
)
|
|
152
|
+
synth.fit(aug)
|
|
153
|
+
self._synths[name] = synth
|
|
154
|
+
pk_gen = IdGenerator(table.columns[pk]).fit(df[pk]) if pk else None
|
|
155
|
+
self._info[name] = _TableInfo(
|
|
156
|
+
n_rows=len(df),
|
|
157
|
+
columns=list(df.columns),
|
|
158
|
+
dtypes=dict(df.dtypes),
|
|
159
|
+
primary_key=pk,
|
|
160
|
+
pk_generator=pk_gen,
|
|
161
|
+
primary=primary,
|
|
162
|
+
secondary=parents[1:],
|
|
163
|
+
self_refs=self_refs,
|
|
164
|
+
context=context,
|
|
165
|
+
null_rates=null_rates,
|
|
166
|
+
)
|
|
167
|
+
self._fitted = True
|
|
168
|
+
return self
|
|
169
|
+
|
|
170
|
+
# ------------------------------------------------------------------ sample
|
|
171
|
+
def sample(
|
|
172
|
+
self,
|
|
173
|
+
scale: float = 1.0,
|
|
174
|
+
seed: int | None = None,
|
|
175
|
+
num_rows: Mapping[str, int] | None = None,
|
|
176
|
+
) -> dict[str, pd.DataFrame]:
|
|
177
|
+
"""Generate every table.
|
|
178
|
+
|
|
179
|
+
``scale`` sizes root tables relative to the training data (child tables follow from
|
|
180
|
+
the learned child counts). ``num_rows`` sets exact row counts for root tables.
|
|
181
|
+
"""
|
|
182
|
+
if not self._fitted:
|
|
183
|
+
raise RuntimeError("Call fit() before sampling")
|
|
184
|
+
rng = np.random.default_rng(seed)
|
|
185
|
+
num_rows = dict(num_rows or {})
|
|
186
|
+
synthetic: dict[str, pd.DataFrame] = {}
|
|
187
|
+
|
|
188
|
+
for name in self._order:
|
|
189
|
+
info, synth = self._info[name], self._synths[name]
|
|
190
|
+
if info.primary is None:
|
|
191
|
+
n = num_rows.get(name, int(round(info.n_rows * scale)) if info.n_rows else 0)
|
|
192
|
+
df = synth._generate(n, rng).reset_index(drop=True)
|
|
193
|
+
else:
|
|
194
|
+
df = self._sample_children(name, info, synth, synthetic, rng)
|
|
195
|
+
if info.primary_key:
|
|
196
|
+
df[info.primary_key] = info.pk_generator.generate(len(df), rng, unique=True, allow_nulls=False)
|
|
197
|
+
for rel in info.secondary:
|
|
198
|
+
df[rel.child_foreign_key] = self._pick_parents(rel, synthetic, len(df), info, rng)
|
|
199
|
+
for rel in info.self_refs:
|
|
200
|
+
keys = df[info.primary_key].to_numpy()
|
|
201
|
+
picked = keys[rng.integers(0, len(keys), size=len(df))].astype(object) if len(keys) else np.array([], dtype=object)
|
|
202
|
+
rate = info.null_rates.get(rel.child_foreign_key, 0.0)
|
|
203
|
+
picked[rng.random(len(df)) < rate] = None
|
|
204
|
+
df[rel.child_foreign_key] = picked
|
|
205
|
+
synthetic[name] = df
|
|
206
|
+
|
|
207
|
+
out = {}
|
|
208
|
+
for name in self._order:
|
|
209
|
+
info = self._info[name]
|
|
210
|
+
df = synthetic[name][info.columns].copy()
|
|
211
|
+
for col in self._keys(name):
|
|
212
|
+
if col in df.columns:
|
|
213
|
+
df[col] = restore_dtype(df[col], info.dtypes[col])
|
|
214
|
+
out[name] = df.reset_index(drop=True)
|
|
215
|
+
return {name: out[name] for name in self.metadata.tables}
|
|
216
|
+
|
|
217
|
+
def _sample_children(self, name, info, synth, synthetic, rng) -> pd.DataFrame:
|
|
218
|
+
rel = info.primary
|
|
219
|
+
parent = synthetic[rel.parent_table]
|
|
220
|
+
counts = (
|
|
221
|
+
pd.to_numeric(parent[_count_col(rel)], errors="coerce").fillna(0).clip(lower=0).round().astype(int).to_numpy()
|
|
222
|
+
)
|
|
223
|
+
rep = np.repeat(np.arange(len(parent)), counts)
|
|
224
|
+
fk_values = parent[rel.parent_primary_key].to_numpy()[rep].astype(object)
|
|
225
|
+
ctx = parent.iloc[rep][info.context].reset_index(drop=True)
|
|
226
|
+
ctx.columns = [_parent_col(c) for c in info.context]
|
|
227
|
+
df = synth._generate(len(rep), rng, ctx if len(ctx.columns) else None, on_fail="drop")
|
|
228
|
+
fk_values = fk_values[df.index.to_numpy()]
|
|
229
|
+
df = df.reset_index(drop=True)
|
|
230
|
+
df[rel.child_foreign_key] = fk_values
|
|
231
|
+
|
|
232
|
+
null_rate = info.null_rates.get(rel.child_foreign_key, 0.0)
|
|
233
|
+
if 0 < null_rate < 1 and len(df):
|
|
234
|
+
extra_n = int(round(len(df) * null_rate / (1 - null_rate)))
|
|
235
|
+
if extra_n:
|
|
236
|
+
extra = synth._generate(extra_n, rng, on_fail="drop").reset_index(drop=True)
|
|
237
|
+
extra[rel.child_foreign_key] = None
|
|
238
|
+
df = pd.concat([df, extra], ignore_index=True)
|
|
239
|
+
return df
|
|
240
|
+
|
|
241
|
+
def _pick_parents(self, rel, synthetic, n, info, rng) -> np.ndarray:
|
|
242
|
+
parent = synthetic[rel.parent_table]
|
|
243
|
+
if len(parent) == 0 or n == 0:
|
|
244
|
+
return np.full(n, None, dtype=object)
|
|
245
|
+
weights = pd.to_numeric(parent[_count_col(rel)], errors="coerce").fillna(0).clip(lower=0).to_numpy(dtype=float)
|
|
246
|
+
p = weights / weights.sum() if weights.sum() > 0 else None
|
|
247
|
+
keys = parent[rel.parent_primary_key].to_numpy()
|
|
248
|
+
picked = keys[rng.choice(len(parent), size=n, p=p)].astype(object)
|
|
249
|
+
picked[rng.random(n) < info.null_rates.get(rel.child_foreign_key, 0.0)] = None
|
|
250
|
+
return picked
|
|
251
|
+
|
|
252
|
+
# ------------------------------------------------------------------ io
|
|
253
|
+
def get_table_synthesizer(self, table: str) -> GaussianCopulaSynthesizer:
|
|
254
|
+
return self._synths[table]
|
|
255
|
+
|
|
256
|
+
def save(self, path: str | Path) -> None:
|
|
257
|
+
"""Save the fitted synthesizer (pickle). Only load files you trust."""
|
|
258
|
+
with open(path, "wb") as fh:
|
|
259
|
+
pickle.dump({"synthschema_version": __version__, "synthesizer": self}, fh)
|
|
260
|
+
|
|
261
|
+
@classmethod
|
|
262
|
+
def load(cls, path: str | Path) -> MultiTableSynthesizer:
|
|
263
|
+
with open(path, "rb") as fh:
|
|
264
|
+
payload = pickle.load(fh) # noqa: S301 - documented: trusted files only
|
|
265
|
+
obj = payload["synthesizer"] if isinstance(payload, dict) else payload
|
|
266
|
+
if not isinstance(obj, cls):
|
|
267
|
+
raise TypeError(f"{path} does not contain a {cls.__name__}")
|
|
268
|
+
return obj
|
|
269
|
+
|
|
270
|
+
def __repr__(self) -> str:
|
|
271
|
+
state = "fitted" if self._fitted else "not fitted"
|
|
272
|
+
return f"MultiTableSynthesizer(tables={list(self.metadata.tables)}, {state})"
|
synthschema/py.typed
ADDED
|
File without changes
|