synthschema 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
synthschema/copula.py ADDED
@@ -0,0 +1,79 @@
1
+ """Multivariate Gaussian copula with exact conditional sampling."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import numpy as np
6
+
7
+
8
+ def _clip_psd(matrix: np.ndarray, floor: float = 1e-8) -> np.ndarray:
9
+ sym = (matrix + matrix.T) / 2
10
+ w, v = np.linalg.eigh(sym)
11
+ return (v * np.clip(w, floor, None)) @ v.T
12
+
13
+
14
+ def nearest_correlation(matrix: np.ndarray) -> np.ndarray:
15
+ fixed = _clip_psd(matrix)
16
+ d = np.sqrt(np.diag(fixed))
17
+ out = fixed / np.outer(d, d)
18
+ np.fill_diagonal(out, 1.0)
19
+ return out
20
+
21
+
22
+ def _cholesky(matrix: np.ndarray) -> np.ndarray:
23
+ jitter = 1e-10
24
+ for _ in range(8):
25
+ try:
26
+ return np.linalg.cholesky(matrix + jitter * np.eye(len(matrix)))
27
+ except np.linalg.LinAlgError:
28
+ jitter *= 100
29
+ return np.linalg.cholesky(_clip_psd(matrix, 1e-6))
30
+
31
+
32
+ class GaussianCopula:
33
+ """Learns the correlation of normal scores and samples from it."""
34
+
35
+ def fit(self, Z: np.ndarray) -> GaussianCopula:
36
+ n, d = Z.shape
37
+ self.dim = d
38
+ if d == 0:
39
+ self.corr = np.zeros((0, 0))
40
+ elif n < 2 or d == 1:
41
+ self.corr = np.eye(d)
42
+ else:
43
+ with np.errstate(invalid="ignore", divide="ignore"):
44
+ corr = np.corrcoef(Z, rowvar=False)
45
+ corr = np.nan_to_num(np.atleast_2d(corr), nan=0.0)
46
+ np.fill_diagonal(corr, 1.0)
47
+ self.corr = nearest_correlation(corr)
48
+ self._chol = _cholesky(self.corr) if d else np.zeros((0, 0))
49
+ self._cond_cache: dict = {}
50
+ return self
51
+
52
+ def sample(self, n: int, rng: np.random.Generator) -> np.ndarray:
53
+ if self.dim == 0:
54
+ return np.zeros((n, 0))
55
+ return rng.standard_normal((n, self.dim)) @ self._chol.T
56
+
57
+ def sample_conditional(
58
+ self, given_idx: list[int], given: np.ndarray, rng: np.random.Generator
59
+ ) -> np.ndarray:
60
+ """Sample the free dimensions given the values of ``given_idx`` (one row per sample)."""
61
+ n = len(given)
62
+ if not given_idx:
63
+ return self.sample(n, rng)
64
+ key = tuple(given_idx)
65
+ if key not in self._cond_cache:
66
+ free = [i for i in range(self.dim) if i not in set(given_idx)]
67
+ s11 = self.corr[np.ix_(free, free)]
68
+ s12 = self.corr[np.ix_(free, given_idx)]
69
+ s22 = self.corr[np.ix_(given_idx, given_idx)]
70
+ a = np.linalg.solve(s22 + 1e-9 * np.eye(len(given_idx)), s12.T).T
71
+ cov = _clip_psd(s11 - a @ s12.T, 1e-10) if free else np.zeros((0, 0))
72
+ chol = _cholesky(cov) if free else cov
73
+ self._cond_cache[key] = (free, a, chol)
74
+ free, a, chol = self._cond_cache[key]
75
+ out = np.empty((n, self.dim))
76
+ out[:, given_idx] = given
77
+ if free:
78
+ out[:, free] = given @ a.T + rng.standard_normal((n, len(free))) @ chol.T
79
+ return out
@@ -0,0 +1,106 @@
1
+ """Demo datasets. These are simulated (no real people or companies) and deterministic."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import numpy as np
6
+ import pandas as pd
7
+
8
+ from .generators import _builtin_pii
9
+ from .metadata import Metadata
10
+
11
+
12
+ def _retail(n_customers: int = 1000, seed: int = 0) -> tuple[dict[str, pd.DataFrame], Metadata]:
13
+ rng = np.random.default_rng(seed)
14
+ n = n_customers
15
+ region = rng.choice(["North", "South", "West"], size=n, p=[0.4, 0.35, 0.25])
16
+ age = np.clip(np.round(rng.normal(42, 12, n)), 18, 85).astype(int)
17
+ seg_p = np.where(age[:, None] > 45, [0.45, 0.35, 0.20], [0.65, 0.15, 0.20])
18
+ segment = np.array(
19
+ [rng.choice(["Consumer", "Corporate", "Small Business"], p=p) for p in seg_p], dtype=object
20
+ )
21
+ seg_mult = pd.Series(segment).map({"Consumer": 1.0, "Corporate": 1.8, "Small Business": 1.4}).to_numpy()
22
+ income = np.round(np.exp(rng.normal(10.6, 0.35, n)) * seg_mult * (0.7 + age / 100), -2)
23
+ signup = pd.Timestamp("2019-01-01") + pd.to_timedelta(rng.integers(0, 6 * 365, n), unit="D")
24
+ first, last = _builtin_pii("first_name", n, rng), _builtin_pii("last_name", n, rng)
25
+ email = [f"{fn.lower()}.{ln.lower()}{i}@example.com" for i, (fn, ln) in enumerate(zip(first, last, strict=True))]
26
+ tenure_days = (pd.Timestamp("2025-01-01") - signup).days.to_numpy()
27
+ active_p = 1 / (1 + np.exp(-(1.5 - tenure_days / 1000 + (income > 60000) * 0.5)))
28
+ customers = pd.DataFrame({
29
+ "customer_id": np.arange(1, n + 1),
30
+ "first_name": first,
31
+ "last_name": last,
32
+ "email": email,
33
+ "age": age,
34
+ "region": region,
35
+ "segment": segment,
36
+ "income": income,
37
+ "signup_date": signup.strftime("%Y-%m-%d"),
38
+ "is_active": rng.random(n) < active_p,
39
+ })
40
+
41
+ lam = 1.5 + (income / income.mean()) * 1.5 + (segment == "Corporate") * 1.0
42
+ counts = rng.poisson(lam)
43
+ owner = np.repeat(np.arange(n), counts)
44
+ m = len(owner)
45
+ order_date = signup[owner] + pd.to_timedelta(rng.integers(0, 365, m), unit="D")
46
+ ship_days = rng.choice([1, 2, 3, 4, 5, 7], size=m, p=[0.15, 0.3, 0.25, 0.15, 0.1, 0.05])
47
+ channel = np.where(age[owner] < 40,
48
+ rng.choice(["Online", "Store", "Phone"], size=m, p=[0.7, 0.25, 0.05]),
49
+ rng.choice(["Online", "Store", "Phone"], size=m, p=[0.4, 0.45, 0.15]))
50
+ amount = np.round(np.exp(rng.normal(3.6, 0.6, m)) * (income[owner] / 50000) ** 0.6 * seg_mult[owner], 2)
51
+ discount = np.round(amount * rng.uniform(0, 0.3, m) * (rng.random(m) < 0.4), 2)
52
+ status = rng.choice(["delivered", "shipped", "returned", "cancelled"], size=m, p=[0.8, 0.1, 0.06, 0.04])
53
+ orders = pd.DataFrame({
54
+ "order_id": [f"ORD-{i:06d}" for i in range(1, m + 1)],
55
+ "customer_id": customers["customer_id"].to_numpy()[owner],
56
+ "order_date": order_date.strftime("%Y-%m-%d"),
57
+ "ship_date": (order_date + pd.to_timedelta(ship_days, unit="D")).strftime("%Y-%m-%d"),
58
+ "channel": channel,
59
+ "status": status,
60
+ "amount": amount,
61
+ "discount": discount,
62
+ })
63
+
64
+ md = Metadata()
65
+ t = md.add_table("customers", primary_key="customer_id")
66
+ md.add_column("customers", "customer_id", "id", representation="int")
67
+ md.add_column("customers", "first_name", "first_name")
68
+ md.add_column("customers", "last_name", "last_name")
69
+ md.add_column("customers", "email", "email")
70
+ md.add_column("customers", "age", "numerical", representation="int")
71
+ md.add_column("customers", "region", "categorical")
72
+ md.add_column("customers", "segment", "categorical")
73
+ md.add_column("customers", "income", "numerical")
74
+ md.add_column("customers", "signup_date", "datetime", datetime_format="%Y-%m-%d")
75
+ md.add_column("customers", "is_active", "boolean")
76
+ t.primary_key = "customer_id"
77
+ md.add_table("orders", primary_key="order_id")
78
+ md.add_column("orders", "order_id", "id", regex_format="ORD-[0-9]{6}")
79
+ md.add_column("orders", "customer_id", "id", representation="int")
80
+ md.add_column("orders", "order_date", "datetime", datetime_format="%Y-%m-%d")
81
+ md.add_column("orders", "ship_date", "datetime", datetime_format="%Y-%m-%d")
82
+ md.add_column("orders", "channel", "categorical")
83
+ md.add_column("orders", "status", "categorical")
84
+ md.add_column("orders", "amount", "numerical")
85
+ md.add_column("orders", "discount", "numerical")
86
+ md.add_relationship("customers", "orders", "customer_id")
87
+ md.add_constraint("orders", "ship_date >= order_date")
88
+ md.add_constraint("orders", "discount <= amount")
89
+ md.add_constraint("orders", "discount >= 0")
90
+ md.validate()
91
+ return {"customers": customers, "orders": orders}, md
92
+
93
+
94
+ def load_demo(name: str = "retail", seed: int = 0):
95
+ """Load a simulated demo dataset.
96
+
97
+ * ``"retail"`` - ``(dict of DataFrames, Metadata)`` with customers and orders.
98
+ * ``"customers"`` - ``(DataFrame, Metadata)`` with the customers table only.
99
+ """
100
+ data, md = _retail(seed=seed)
101
+ if name == "retail":
102
+ return data, md
103
+ if name == "customers":
104
+ single = Metadata.from_dict({"tables": {"customers": md.tables["customers"].to_dict()}})
105
+ return data["customers"], single
106
+ raise ValueError(f"Unknown demo {name!r}; choose 'retail' or 'customers'")
@@ -0,0 +1,446 @@
1
+ """Evaluation: is the synthetic data valid, faithful and private?
2
+
3
+ * :func:`evaluate_quality` - statistical fidelity (column shapes, column-pair trends,
4
+ and child-count shapes for multi-table data). Scores are in [0, 1]; higher is better.
5
+ * :func:`run_diagnostic` - structural validity: keys, foreign keys, ranges, categories, rules.
6
+ * :func:`evaluate_privacy` - copies of real rows and distance to closest record (DCR).
7
+ * :func:`certify` - combines the three into a pass/fail :class:`Certificate` with hard stops.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import hashlib
13
+ import json
14
+ from collections.abc import Mapping
15
+ from dataclasses import dataclass, field
16
+ from datetime import datetime, timezone
17
+ from itertools import combinations
18
+ from pathlib import Path
19
+ from typing import Any
20
+
21
+ import numpy as np
22
+ import pandas as pd
23
+ from scipy import stats
24
+
25
+ from ._version import __version__
26
+ from .constraints import build_constraints
27
+ from .metadata import Metadata, Table
28
+ from .transformers import datetime_to_seconds
29
+
30
+ Data = pd.DataFrame | Mapping[str, pd.DataFrame]
31
+
32
+
33
+ def _as_tables(real: Data, synthetic: Data, metadata: Metadata | None) -> tuple[dict, dict, Metadata]:
34
+ if isinstance(real, pd.DataFrame):
35
+ if metadata is None:
36
+ metadata = Metadata.detect_from_dataframe(real)
37
+ name = metadata.get_table().name
38
+ return {name: real}, {name: synthetic}, metadata
39
+ if metadata is None:
40
+ metadata = Metadata.detect_from_dataframes(real)
41
+ return dict(real), dict(synthetic), metadata
42
+
43
+
44
+ def _numeric(series: pd.Series, table: Table, col: str) -> np.ndarray:
45
+ meta = table.columns.get(col)
46
+ if meta is not None and meta.sdtype == "datetime":
47
+ return datetime_to_seconds(series, meta.datetime_format)
48
+ return pd.to_numeric(series, errors="coerce").astype(float).to_numpy()
49
+
50
+
51
+ def _is_numeric(table: Table, col: str) -> bool:
52
+ meta = table.columns.get(col)
53
+ return meta is not None and meta.sdtype in {"numerical", "datetime"}
54
+
55
+
56
+ def _tvd_score(a: pd.Series, b: pd.Series) -> float:
57
+ pa = a.astype(object).where(a.notna(), "__null__").value_counts(normalize=True)
58
+ pb = b.astype(object).where(b.notna(), "__null__").value_counts(normalize=True)
59
+ idx = pa.index.union(pb.index)
60
+ return float(1 - 0.5 * np.abs(pa.reindex(idx, fill_value=0) - pb.reindex(idx, fill_value=0)).sum())
61
+
62
+
63
+ def _ks_score(a: np.ndarray, b: np.ndarray) -> float:
64
+ a, b = a[np.isfinite(a)], b[np.isfinite(b)]
65
+ if len(a) == 0 or len(b) == 0:
66
+ return float(len(a) == len(b))
67
+ return float(1 - stats.ks_2samp(a, b, method="asymp").statistic)
68
+
69
+
70
+ def _binned(x: np.ndarray, edges: np.ndarray) -> pd.Series:
71
+ out = np.digitize(x, edges[1:-1]).astype(object)
72
+ out[~np.isfinite(x)] = "__null__"
73
+ return pd.Series(out)
74
+
75
+
76
+ @dataclass
77
+ class QualityReport:
78
+ """Statistical fidelity. ``score`` is the mean of the property scores."""
79
+
80
+ score: float
81
+ properties: dict[str, float]
82
+ details: dict[str, pd.DataFrame] = field(default_factory=dict)
83
+
84
+ def to_dict(self) -> dict:
85
+ return {"score": self.score, "properties": self.properties}
86
+
87
+ def __repr__(self) -> str:
88
+ props = ", ".join(f"{k}: {v:.1%}" for k, v in self.properties.items())
89
+ return f"QualityReport(score={self.score:.1%}; {props})"
90
+
91
+
92
+ def _table_quality(real: pd.DataFrame, synth: pd.DataFrame, table: Table, keys: set[str], max_pair_columns: int):
93
+ cols = [c for c in table.modeled_columns if c in real.columns and c in synth.columns and c not in keys]
94
+ shape_rows = []
95
+ for col in cols:
96
+ if _is_numeric(table, col):
97
+ score = _ks_score(_numeric(real[col], table, col), _numeric(synth[col], table, col))
98
+ metric = "KSComplement"
99
+ else:
100
+ score = _tvd_score(real[col], synth[col])
101
+ metric = "TVComplement"
102
+ shape_rows.append({"table": table.name, "column": col, "metric": metric, "score": score})
103
+
104
+ pair_rows = []
105
+ pair_cols = cols[:max_pair_columns]
106
+ prepared = {}
107
+ for col in pair_cols:
108
+ if _is_numeric(table, col):
109
+ r, s = _numeric(real[col], table, col), _numeric(synth[col], table, col)
110
+ finite = r[np.isfinite(r)]
111
+ edges = np.unique(np.quantile(finite, np.linspace(0, 1, 11))) if len(finite) else np.array([0, 1])
112
+ prepared[col] = ("num", r, s, _binned(r, edges), _binned(s, edges))
113
+ else:
114
+ r = real[col].astype(object).where(real[col].notna(), "__null__").reset_index(drop=True)
115
+ s = synth[col].astype(object).where(synth[col].notna(), "__null__").reset_index(drop=True)
116
+ prepared[col] = ("cat", None, None, r, s)
117
+ for a, b in combinations(pair_cols, 2):
118
+ ka, ra, sa, rba, sba = prepared[a]
119
+ kb, rb, sb, rbb, sbb = prepared[b]
120
+ if ka == kb == "num":
121
+ rc = pd.Series(ra).corr(pd.Series(rb))
122
+ sc = pd.Series(sa).corr(pd.Series(sb))
123
+ if np.isnan(rc) or np.isnan(sc):
124
+ continue
125
+ score, metric = float(1 - abs(rc - sc) / 2), "CorrelationSimilarity"
126
+ else:
127
+ jr = (rba.astype(str) + "\x1f" + rbb.astype(str)).value_counts(normalize=True)
128
+ js = (sba.astype(str) + "\x1f" + sbb.astype(str)).value_counts(normalize=True)
129
+ idx = jr.index.union(js.index)
130
+ score = float(1 - 0.5 * np.abs(jr.reindex(idx, fill_value=0) - js.reindex(idx, fill_value=0)).sum())
131
+ metric = "ContingencySimilarity"
132
+ pair_rows.append({"table": table.name, "column_1": a, "column_2": b, "metric": metric, "score": score})
133
+ return shape_rows, pair_rows
134
+
135
+
136
+ def evaluate_quality(
137
+ real: Data, synthetic: Data, metadata: Metadata | None = None, max_pair_columns: int = 40
138
+ ) -> QualityReport:
139
+ """Score how closely the synthetic data reproduces the real data's statistics."""
140
+ real_t, synth_t, md = _as_tables(real, synthetic, metadata)
141
+ shapes, pairs, cards = [], [], []
142
+ for name, table in md.tables.items():
143
+ keys = set(md.foreign_keys(name)) | ({table.primary_key} if table.primary_key else set())
144
+ s, p = _table_quality(real_t[name], synth_t[name], table, keys, max_pair_columns)
145
+ shapes.extend(s)
146
+ pairs.extend(p)
147
+ for rel in md.relationships:
148
+ rc = real_t[rel.child_table][rel.child_foreign_key].value_counts()
149
+ sc = synth_t[rel.child_table][rel.child_foreign_key].value_counts()
150
+ r_counts = real_t[rel.parent_table][rel.parent_primary_key].map(rc).fillna(0).to_numpy(dtype=float)
151
+ s_counts = synth_t[rel.parent_table][rel.parent_primary_key].map(sc).fillna(0).to_numpy(dtype=float)
152
+ cards.append({
153
+ "parent_table": rel.parent_table, "child_table": rel.child_table,
154
+ "metric": "CardinalityShape", "score": _ks_score(r_counts, s_counts),
155
+ })
156
+ details = {
157
+ "Column Shapes": pd.DataFrame(shapes),
158
+ "Column Pair Trends": pd.DataFrame(pairs),
159
+ }
160
+ properties = {
161
+ "Column Shapes": float(np.mean([r["score"] for r in shapes])) if shapes else 1.0,
162
+ "Column Pair Trends": float(np.mean([r["score"] for r in pairs])) if pairs else 1.0,
163
+ }
164
+ if cards:
165
+ details["Cardinality"] = pd.DataFrame(cards)
166
+ properties["Cardinality"] = float(np.mean([r["score"] for r in cards]))
167
+ return QualityReport(float(np.mean(list(properties.values()))), properties, details)
168
+
169
+
170
+ # ---------------------------------------------------------------------- diagnostic
171
+
172
+
173
+ @dataclass
174
+ class DiagnosticReport:
175
+ """Structural checks. ``passed`` is False if any hard check fails."""
176
+
177
+ checks: pd.DataFrame
178
+
179
+ @property
180
+ def passed(self) -> bool:
181
+ hard = self.checks[self.checks["hard"]]
182
+ return bool((hard["score"] >= 1.0).all())
183
+
184
+ @property
185
+ def score(self) -> float:
186
+ return float(self.checks["score"].mean()) if len(self.checks) else 1.0
187
+
188
+ def failures(self) -> pd.DataFrame:
189
+ return self.checks[self.checks["score"] < 1.0]
190
+
191
+ def to_dict(self) -> dict:
192
+ return {"passed": self.passed, "score": self.score, "checks": self.checks.to_dict("records")}
193
+
194
+ def __repr__(self) -> str:
195
+ return f"DiagnosticReport(passed={self.passed}, score={self.score:.1%}, checks={len(self.checks)})"
196
+
197
+
198
+ def run_diagnostic(real: Data, synthetic: Data, metadata: Metadata | None = None) -> DiagnosticReport:
199
+ """Check keys, foreign keys, column presence, ranges, categories and rules."""
200
+ real_t, synth_t, md = _as_tables(real, synthetic, metadata)
201
+ rows: list[dict[str, Any]] = []
202
+
203
+ def add(table, check, column, score, hard):
204
+ rows.append({"table": table, "check": check, "column": column, "score": float(score), "hard": hard})
205
+
206
+ for name, table in md.tables.items():
207
+ r, s = real_t[name], synth_t[name]
208
+ missing = [c for c in r.columns if c not in s.columns]
209
+ add(name, "columns present", None, 1.0 - len(missing) / max(1, len(r.columns)), True)
210
+ if table.primary_key and table.primary_key in s.columns:
211
+ pk = s[table.primary_key]
212
+ add(name, "primary key unique and non-null", table.primary_key,
213
+ float(pk.notna().all() and pk.is_unique), True)
214
+ keys = set(md.foreign_keys(name)) | ({table.primary_key} if table.primary_key else set())
215
+ for col in table.modeled_columns:
216
+ if col in keys or col not in s.columns:
217
+ continue
218
+ if _is_numeric(table, col):
219
+ rv, sv = _numeric(r[col], table, col), _numeric(s[col], table, col)
220
+ rv, sv = rv[np.isfinite(rv)], sv[np.isfinite(sv)]
221
+ if len(rv) and len(sv):
222
+ within = ((sv >= rv.min()) & (sv <= rv.max())).mean()
223
+ add(name, "within real min/max", col, within, False)
224
+ else:
225
+ allowed = set(r[col].dropna().unique())
226
+ sv = s[col].dropna()
227
+ if len(sv):
228
+ add(name, "categories seen in real data", col, sv.isin(allowed).mean(), False)
229
+ for constraint in build_constraints(table.constraints):
230
+ if all(c in s.columns for c in constraint.columns):
231
+ constraint.fit(r, table)
232
+ add(name, f"rule {constraint.to_dict()}", None, constraint.is_valid(s).mean(), True)
233
+ for rel in md.relationships:
234
+ fk = synth_t[rel.child_table][rel.child_foreign_key].dropna()
235
+ parent_keys = set(synth_t[rel.parent_table][rel.parent_primary_key].dropna())
236
+ score = fk.isin(parent_keys).mean() if len(fk) else 1.0
237
+ add(rel.child_table, f"foreign key -> {rel.parent_table}", rel.child_foreign_key, score, True)
238
+ return DiagnosticReport(pd.DataFrame(rows, columns=["table", "check", "column", "score", "hard"]))
239
+
240
+
241
+ # ---------------------------------------------------------------------- privacy
242
+
243
+
244
+ @dataclass
245
+ class PrivacyReport:
246
+ exact_match_rate: float
247
+ unique_match_rate: float
248
+ dcr_median: float | None
249
+ baseline_dcr_median: float | None
250
+ details: dict[str, Any] = field(default_factory=dict)
251
+
252
+ @property
253
+ def dcr_ratio(self) -> float | None:
254
+ if self.dcr_median is None or not self.baseline_dcr_median:
255
+ return None
256
+ return self.dcr_median / self.baseline_dcr_median
257
+
258
+ def to_dict(self) -> dict:
259
+ return {
260
+ "exact_match_rate": self.exact_match_rate,
261
+ "unique_match_rate": self.unique_match_rate,
262
+ "dcr_median": self.dcr_median,
263
+ "baseline_dcr_median": self.baseline_dcr_median,
264
+ "dcr_ratio": self.dcr_ratio,
265
+ **self.details,
266
+ }
267
+
268
+ def __repr__(self) -> str:
269
+ ratio = "n/a" if self.dcr_ratio is None else f"{self.dcr_ratio:.2f}"
270
+ return (
271
+ f"PrivacyReport(exact_match_rate={self.exact_match_rate:.2%}, "
272
+ f"unique_match_rate={self.unique_match_rate:.2%}, dcr_ratio={ratio})"
273
+ )
274
+
275
+
276
+ def _encode(df: pd.DataFrame, table: Table, cols: list[str], ref: pd.DataFrame) -> np.ndarray:
277
+ parts = []
278
+ for col in cols:
279
+ if _is_numeric(table, col):
280
+ r = _numeric(ref[col], table, col)
281
+ x = _numeric(df[col], table, col)
282
+ mu, sd = np.nanmean(r), np.nanstd(r) or 1.0
283
+ parts.append(np.nan_to_num((x - mu) / sd)[:, None])
284
+ else:
285
+ top = ref[col].astype(str).value_counts().index[:20]
286
+ vals = df[col].astype(str).to_numpy()
287
+ parts.append(np.stack([(vals == t).astype(float) for t in top], axis=1) if len(top) else np.zeros((len(df), 0)))
288
+ return np.hstack(parts) if parts else np.zeros((len(df), 0))
289
+
290
+
291
+ def _min_dist(a: np.ndarray, b: np.ndarray, exclude_self: bool = False) -> np.ndarray:
292
+ out = np.empty(len(a))
293
+ b2 = (b**2).sum(axis=1)
294
+ for start in range(0, len(a), 512):
295
+ chunk = a[start:start + 512]
296
+ d = (chunk**2).sum(axis=1)[:, None] + b2[None, :] - 2 * chunk @ b.T
297
+ d = np.sqrt(np.clip(d, 0, None))
298
+ if exclude_self:
299
+ idx = np.arange(start, start + len(chunk))
300
+ d[np.arange(len(chunk)), idx] = np.inf
301
+ out[start:start + len(chunk)] = d.min(axis=1)
302
+ return out
303
+
304
+
305
+ def evaluate_privacy(
306
+ real: pd.DataFrame,
307
+ synthetic: pd.DataFrame,
308
+ metadata: Metadata | None = None,
309
+ table_name: str | None = None,
310
+ max_rows: int = 2000,
311
+ seed: int = 0,
312
+ ) -> PrivacyReport:
313
+ """Measure copying of real rows (single table).
314
+
315
+ * ``exact_match_rate`` - share of synthetic rows identical to some real row (modelled columns).
316
+ * ``unique_match_rate`` - share of synthetic rows identical to a real row that is *unique*
317
+ in the real data. This is the re-identification risk; it should be 0.
318
+ * DCR - median distance from synthetic rows to their closest real row, compared with the
319
+ same distance between real rows (``dcr_ratio`` near or above 1 means no memorisation).
320
+ """
321
+ if metadata is None:
322
+ metadata = Metadata.detect_from_dataframe(real)
323
+ table = metadata.get_table(table_name)
324
+ keys = {table.primary_key} | set(metadata.foreign_keys(table.name))
325
+ cols = [c for c in table.modeled_columns if c in synthetic.columns and c not in keys]
326
+ if not cols:
327
+ return PrivacyReport(0.0, 0.0, None, None, {"columns": []})
328
+ r_hash = pd.util.hash_pandas_object(real[cols].astype(str), index=False)
329
+ s_hash = pd.util.hash_pandas_object(synthetic[cols].astype(str), index=False)
330
+ counts = r_hash.value_counts()
331
+ matched = s_hash.isin(counts.index)
332
+ unique_hashes = set(counts[counts == 1].index)
333
+ exact = float(matched.mean()) if len(s_hash) else 0.0
334
+ unique = float(s_hash.isin(unique_hashes).mean()) if len(s_hash) else 0.0
335
+
336
+ rng = np.random.default_rng(seed)
337
+ r_idx = rng.choice(len(real), min(len(real), max_rows * 2), replace=False)
338
+ s_idx = rng.choice(len(synthetic), min(len(synthetic), max_rows), replace=False)
339
+ ref = real.iloc[r_idx].reset_index(drop=True)
340
+ R = _encode(ref, table, cols, real)
341
+ S = _encode(synthetic.iloc[s_idx].reset_index(drop=True), table, cols, real)
342
+ dcr = baseline = None
343
+ if R.shape[1] and len(R) > 1 and len(S):
344
+ dcr = float(np.median(_min_dist(S, R)))
345
+ half = min(len(R), max_rows)
346
+ baseline = float(np.median(_min_dist(R[:half], R, exclude_self=True)))
347
+ return PrivacyReport(exact, unique, dcr, baseline, {"columns": cols})
348
+
349
+
350
+ # ---------------------------------------------------------------------- certificate
351
+
352
+
353
+ @dataclass
354
+ class Certificate:
355
+ """Pass/fail release decision with the evidence behind it."""
356
+
357
+ passed: bool
358
+ reasons: list[str]
359
+ quality: QualityReport
360
+ diagnostic: DiagnosticReport
361
+ privacy: dict[str, PrivacyReport]
362
+ thresholds: dict[str, float]
363
+ fingerprint: str
364
+ created_at: str
365
+ version: str = __version__
366
+
367
+ def to_dict(self) -> dict:
368
+ return {
369
+ "passed": self.passed,
370
+ "reasons": self.reasons,
371
+ "quality": self.quality.to_dict(),
372
+ "diagnostic": {"passed": self.diagnostic.passed, "score": self.diagnostic.score,
373
+ "failures": self.diagnostic.failures().to_dict("records")},
374
+ "privacy": {k: v.to_dict() for k, v in self.privacy.items()},
375
+ "thresholds": self.thresholds,
376
+ "fingerprint": self.fingerprint,
377
+ "created_at": self.created_at,
378
+ "synthschema_version": self.version,
379
+ }
380
+
381
+ def to_json(self, indent: int = 2) -> str:
382
+ return json.dumps(self.to_dict(), indent=indent, default=str)
383
+
384
+ def save(self, path: str | Path) -> None:
385
+ Path(path).write_text(self.to_json())
386
+
387
+ def __repr__(self) -> str:
388
+ status = "PASSED" if self.passed else "FAILED"
389
+ return (
390
+ f"Certificate({status}; quality={self.quality.score:.1%}, "
391
+ f"diagnostic={'ok' if self.diagnostic.passed else 'failed'}, reasons={self.reasons})"
392
+ )
393
+
394
+
395
+ def _fingerprint(synth_t: Mapping[str, pd.DataFrame], md: Metadata) -> str:
396
+ h = hashlib.sha256()
397
+ h.update(json.dumps(md.to_dict(), sort_keys=True, default=str).encode())
398
+ for name in sorted(synth_t):
399
+ h.update(name.encode())
400
+ h.update(pd.util.hash_pandas_object(synth_t[name].astype(str), index=False).to_numpy().tobytes())
401
+ return h.hexdigest()
402
+
403
+
404
+ def certify(
405
+ real: Data,
406
+ synthetic: Data,
407
+ metadata: Metadata | None = None,
408
+ *,
409
+ min_quality: float = 0.80,
410
+ max_unique_match_rate: float = 0.0,
411
+ min_dcr_ratio: float | None = None,
412
+ ) -> Certificate:
413
+ """Decide whether synthetic data may be released.
414
+
415
+ Hard stops: any failed structural check (keys, foreign keys, rules), any copy of a
416
+ real row that is unique in the real data above ``max_unique_match_rate``, or quality
417
+ below ``min_quality``.
418
+ """
419
+ real_t, synth_t, md = _as_tables(real, synthetic, metadata)
420
+ quality = evaluate_quality(real_t, synth_t, md)
421
+ diagnostic = run_diagnostic(real_t, synth_t, md)
422
+ privacy = {name: evaluate_privacy(real_t[name], synth_t[name], md, name) for name in md.tables}
423
+ reasons = []
424
+ if not diagnostic.passed:
425
+ for row in diagnostic.failures().query("hard").to_dict("records"):
426
+ reasons.append(f"structure: {row['table']}: {row['check']} ({row['score']:.1%})")
427
+ if quality.score < min_quality:
428
+ reasons.append(f"quality {quality.score:.1%} is below {min_quality:.0%}")
429
+ for name, rep in privacy.items():
430
+ if rep.unique_match_rate > max_unique_match_rate:
431
+ reasons.append(f"privacy: {name}: {rep.unique_match_rate:.2%} of rows copy a unique real row")
432
+ if min_dcr_ratio is not None and rep.dcr_ratio is not None and rep.dcr_ratio < min_dcr_ratio:
433
+ reasons.append(f"privacy: {name}: DCR ratio {rep.dcr_ratio:.2f} is below {min_dcr_ratio}")
434
+ thresholds = {"min_quality": min_quality, "max_unique_match_rate": max_unique_match_rate}
435
+ if min_dcr_ratio is not None:
436
+ thresholds["min_dcr_ratio"] = min_dcr_ratio
437
+ return Certificate(
438
+ passed=not reasons,
439
+ reasons=reasons,
440
+ quality=quality,
441
+ diagnostic=diagnostic,
442
+ privacy=privacy,
443
+ thresholds=thresholds,
444
+ fingerprint=_fingerprint(synth_t, md),
445
+ created_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
446
+ )