synthschema 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,272 @@
1
+ """Multi-table synthesis: parent-first, conditional generation that keeps keys intact.
2
+
3
+ Tables are generated in topological order. For each relationship the parent's model also
4
+ learns *how many children* each parent row has (correlated with the parent's attributes),
5
+ and the child's model learns its columns *conditioned on the parent row*. This:
6
+
7
+ * keeps referential integrity by construction (no orphan foreign keys),
8
+ * keeps the child-count distribution per parent,
9
+ * keeps cross-table patterns (for example, high-income customers placing larger orders),
10
+ * scales linearly with the number of tables.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import pickle
16
+ from collections.abc import Mapping
17
+ from dataclasses import dataclass, field
18
+ from pathlib import Path
19
+
20
+ import numpy as np
21
+ import pandas as pd
22
+
23
+ from ._version import __version__
24
+ from .generators import IdGenerator
25
+ from .metadata import Column, Metadata, Relationship, detect_column
26
+ from .single_table import GaussianCopulaSynthesizer
27
+ from .transformers import restore_dtype
28
+
29
+
30
+ def _count_col(rel: Relationship) -> str:
31
+ return f"__count__{rel.child_table}__{rel.child_foreign_key}"
32
+
33
+
34
+ def _parent_col(col: str) -> str:
35
+ return f"__parent__{col}"
36
+
37
+
38
+ @dataclass
39
+ class _TableInfo:
40
+ n_rows: int
41
+ columns: list[str]
42
+ dtypes: dict
43
+ primary_key: str | None
44
+ pk_generator: IdGenerator | None
45
+ primary: Relationship | None
46
+ secondary: list[Relationship] = field(default_factory=list)
47
+ self_refs: list[Relationship] = field(default_factory=list)
48
+ context: list[str] = field(default_factory=list)
49
+ null_rates: dict[str, float] = field(default_factory=dict)
50
+
51
+
52
+ class MultiTableSynthesizer:
53
+ """Synthesizes a set of related tables.
54
+
55
+ Example::
56
+
57
+ metadata = Metadata.detect_from_dataframes({"customers": customers, "orders": orders})
58
+ synth = MultiTableSynthesizer(metadata)
59
+ synth.fit({"customers": customers, "orders": orders})
60
+ fake = synth.sample(scale=1.0, seed=7) # dict of DataFrames
61
+ """
62
+
63
+ def __init__(
64
+ self,
65
+ metadata: Metadata,
66
+ *,
67
+ default_distribution: str = "empirical",
68
+ numerical_distributions: Mapping[str, Mapping[str, str]] | None = None,
69
+ locale: str | list[str] | None = None,
70
+ max_context_columns: int = 20,
71
+ ) -> None:
72
+ self.metadata = metadata
73
+ self.default_distribution = default_distribution
74
+ self.numerical_distributions = dict(numerical_distributions or {})
75
+ self.locale = locale
76
+ self.max_context_columns = max_context_columns
77
+ self._fitted = False
78
+
79
+ def _keys(self, table: str) -> set[str]:
80
+ t = self.metadata.tables[table]
81
+ keys = {r.child_foreign_key for r in self.metadata.relationships if r.child_table == table}
82
+ if t.primary_key:
83
+ keys.add(t.primary_key)
84
+ return keys
85
+
86
+ def _context_columns(self, table: str) -> list[str]:
87
+ t = self.metadata.tables[table]
88
+ keys = self._keys(table)
89
+ cols = [c for c, meta in t.columns.items() if meta.is_modeled and c not in keys]
90
+ return cols[: self.max_context_columns]
91
+
92
+ # ------------------------------------------------------------------ fit
93
+ def fit(self, data: Mapping[str, pd.DataFrame]) -> MultiTableSynthesizer:
94
+ md = self.metadata
95
+ md.validate()
96
+ md.validate_data(data)
97
+ self._order = md.topological_order()
98
+ self._synths: dict[str, GaussianCopulaSynthesizer] = {}
99
+ self._info: dict[str, _TableInfo] = {}
100
+
101
+ for name in self._order:
102
+ table = md.tables[name]
103
+ df = data[name].reset_index(drop=True)
104
+ pk = table.primary_key
105
+ if pk is not None and (df[pk].isna().any() or not df[pk].is_unique):
106
+ raise ValueError(f"Primary key {name}.{pk} must be unique and non-null")
107
+ parents = md.parent_relationships(name)
108
+ children = md.child_relationships(name)
109
+ self_refs = [r for r in md.relationships if r.parent_table == r.child_table == name]
110
+ keys = self._keys(name)
111
+
112
+ own = [c for c in df.columns if c not in keys]
113
+ aug = df[own].copy()
114
+ aug_meta: dict[str, Column] = {
115
+ c: table.columns[c] if c in table.columns else detect_column(df[c], c) for c in own
116
+ }
117
+ for rel in children:
118
+ if pk is None:
119
+ raise ValueError(f"Table {name!r} has children but no primary key")
120
+ counts = data[rel.child_table][rel.child_foreign_key].value_counts()
121
+ aug[_count_col(rel)] = df[pk].map(counts).fillna(0).astype(int).to_numpy()
122
+ aug_meta[_count_col(rel)] = Column("numerical", representation="int")
123
+
124
+ primary = parents[0] if parents else None
125
+ null_rates: dict[str, float] = {}
126
+ context: list[str] = []
127
+ if primary is not None:
128
+ ptable = md.tables[primary.parent_table]
129
+ pdf = data[primary.parent_table]
130
+ context = self._context_columns(primary.parent_table)
131
+ lookup = pdf.set_index(ptable.primary_key)[context]
132
+ fk = df[primary.child_foreign_key]
133
+ joined = lookup.reindex(fk.to_numpy())
134
+ for c in context:
135
+ aug[_parent_col(c)] = joined[c].to_numpy()
136
+ aug_meta[_parent_col(c)] = ptable.columns[c]
137
+ null_rates[primary.child_foreign_key] = float(1 - fk.isin(pdf[ptable.primary_key]).mean())
138
+ for rel in parents[1:] + self_refs:
139
+ null_rates[rel.child_foreign_key] = float(df[rel.child_foreign_key].isna().mean())
140
+
141
+ sub = Metadata()
142
+ sub_table = sub.add_table(name)
143
+ sub_table.columns = aug_meta
144
+ sub_table.constraints = list(table.constraints)
145
+ synth = GaussianCopulaSynthesizer(
146
+ sub,
147
+ name,
148
+ default_distribution=self.default_distribution,
149
+ numerical_distributions=self.numerical_distributions.get(name),
150
+ locale=self.locale,
151
+ )
152
+ synth.fit(aug)
153
+ self._synths[name] = synth
154
+ pk_gen = IdGenerator(table.columns[pk]).fit(df[pk]) if pk else None
155
+ self._info[name] = _TableInfo(
156
+ n_rows=len(df),
157
+ columns=list(df.columns),
158
+ dtypes=dict(df.dtypes),
159
+ primary_key=pk,
160
+ pk_generator=pk_gen,
161
+ primary=primary,
162
+ secondary=parents[1:],
163
+ self_refs=self_refs,
164
+ context=context,
165
+ null_rates=null_rates,
166
+ )
167
+ self._fitted = True
168
+ return self
169
+
170
+ # ------------------------------------------------------------------ sample
171
+ def sample(
172
+ self,
173
+ scale: float = 1.0,
174
+ seed: int | None = None,
175
+ num_rows: Mapping[str, int] | None = None,
176
+ ) -> dict[str, pd.DataFrame]:
177
+ """Generate every table.
178
+
179
+ ``scale`` sizes root tables relative to the training data (child tables follow from
180
+ the learned child counts). ``num_rows`` sets exact row counts for root tables.
181
+ """
182
+ if not self._fitted:
183
+ raise RuntimeError("Call fit() before sampling")
184
+ rng = np.random.default_rng(seed)
185
+ num_rows = dict(num_rows or {})
186
+ synthetic: dict[str, pd.DataFrame] = {}
187
+
188
+ for name in self._order:
189
+ info, synth = self._info[name], self._synths[name]
190
+ if info.primary is None:
191
+ n = num_rows.get(name, int(round(info.n_rows * scale)) if info.n_rows else 0)
192
+ df = synth._generate(n, rng).reset_index(drop=True)
193
+ else:
194
+ df = self._sample_children(name, info, synth, synthetic, rng)
195
+ if info.primary_key:
196
+ df[info.primary_key] = info.pk_generator.generate(len(df), rng, unique=True, allow_nulls=False)
197
+ for rel in info.secondary:
198
+ df[rel.child_foreign_key] = self._pick_parents(rel, synthetic, len(df), info, rng)
199
+ for rel in info.self_refs:
200
+ keys = df[info.primary_key].to_numpy()
201
+ picked = keys[rng.integers(0, len(keys), size=len(df))].astype(object) if len(keys) else np.array([], dtype=object)
202
+ rate = info.null_rates.get(rel.child_foreign_key, 0.0)
203
+ picked[rng.random(len(df)) < rate] = None
204
+ df[rel.child_foreign_key] = picked
205
+ synthetic[name] = df
206
+
207
+ out = {}
208
+ for name in self._order:
209
+ info = self._info[name]
210
+ df = synthetic[name][info.columns].copy()
211
+ for col in self._keys(name):
212
+ if col in df.columns:
213
+ df[col] = restore_dtype(df[col], info.dtypes[col])
214
+ out[name] = df.reset_index(drop=True)
215
+ return {name: out[name] for name in self.metadata.tables}
216
+
217
+ def _sample_children(self, name, info, synth, synthetic, rng) -> pd.DataFrame:
218
+ rel = info.primary
219
+ parent = synthetic[rel.parent_table]
220
+ counts = (
221
+ pd.to_numeric(parent[_count_col(rel)], errors="coerce").fillna(0).clip(lower=0).round().astype(int).to_numpy()
222
+ )
223
+ rep = np.repeat(np.arange(len(parent)), counts)
224
+ fk_values = parent[rel.parent_primary_key].to_numpy()[rep].astype(object)
225
+ ctx = parent.iloc[rep][info.context].reset_index(drop=True)
226
+ ctx.columns = [_parent_col(c) for c in info.context]
227
+ df = synth._generate(len(rep), rng, ctx if len(ctx.columns) else None, on_fail="drop")
228
+ fk_values = fk_values[df.index.to_numpy()]
229
+ df = df.reset_index(drop=True)
230
+ df[rel.child_foreign_key] = fk_values
231
+
232
+ null_rate = info.null_rates.get(rel.child_foreign_key, 0.0)
233
+ if 0 < null_rate < 1 and len(df):
234
+ extra_n = int(round(len(df) * null_rate / (1 - null_rate)))
235
+ if extra_n:
236
+ extra = synth._generate(extra_n, rng, on_fail="drop").reset_index(drop=True)
237
+ extra[rel.child_foreign_key] = None
238
+ df = pd.concat([df, extra], ignore_index=True)
239
+ return df
240
+
241
+ def _pick_parents(self, rel, synthetic, n, info, rng) -> np.ndarray:
242
+ parent = synthetic[rel.parent_table]
243
+ if len(parent) == 0 or n == 0:
244
+ return np.full(n, None, dtype=object)
245
+ weights = pd.to_numeric(parent[_count_col(rel)], errors="coerce").fillna(0).clip(lower=0).to_numpy(dtype=float)
246
+ p = weights / weights.sum() if weights.sum() > 0 else None
247
+ keys = parent[rel.parent_primary_key].to_numpy()
248
+ picked = keys[rng.choice(len(parent), size=n, p=p)].astype(object)
249
+ picked[rng.random(n) < info.null_rates.get(rel.child_foreign_key, 0.0)] = None
250
+ return picked
251
+
252
+ # ------------------------------------------------------------------ io
253
+ def get_table_synthesizer(self, table: str) -> GaussianCopulaSynthesizer:
254
+ return self._synths[table]
255
+
256
+ def save(self, path: str | Path) -> None:
257
+ """Save the fitted synthesizer (pickle). Only load files you trust."""
258
+ with open(path, "wb") as fh:
259
+ pickle.dump({"synthschema_version": __version__, "synthesizer": self}, fh)
260
+
261
+ @classmethod
262
+ def load(cls, path: str | Path) -> MultiTableSynthesizer:
263
+ with open(path, "rb") as fh:
264
+ payload = pickle.load(fh) # noqa: S301 - documented: trusted files only
265
+ obj = payload["synthesizer"] if isinstance(payload, dict) else payload
266
+ if not isinstance(obj, cls):
267
+ raise TypeError(f"{path} does not contain a {cls.__name__}")
268
+ return obj
269
+
270
+ def __repr__(self) -> str:
271
+ state = "fitted" if self._fitted else "not fitted"
272
+ return f"MultiTableSynthesizer(tables={list(self.metadata.tables)}, {state})"
synthschema/py.typed ADDED
File without changes