synthschema 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synthschema/__init__.py +56 -0
- synthschema/_version.py +1 -0
- synthschema/cli.py +134 -0
- synthschema/constraints.py +438 -0
- synthschema/copula.py +79 -0
- synthschema/datasets.py +106 -0
- synthschema/evaluation.py +446 -0
- synthschema/generators.py +456 -0
- synthschema/marginals.py +143 -0
- synthschema/metadata.py +547 -0
- synthschema/multi_table.py +272 -0
- synthschema/py.typed +0 -0
- synthschema/schema.py +490 -0
- synthschema/single_table.py +311 -0
- synthschema/transformers.py +247 -0
- synthschema-0.1.0.dist-info/METADATA +253 -0
- synthschema-0.1.0.dist-info/RECORD +20 -0
- synthschema-0.1.0.dist-info/WHEEL +4 -0
- synthschema-0.1.0.dist-info/entry_points.txt +2 -0
- synthschema-0.1.0.dist-info/licenses/LICENSE +201 -0
synthschema/copula.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Multivariate Gaussian copula with exact conditional sampling."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _clip_psd(matrix: np.ndarray, floor: float = 1e-8) -> np.ndarray:
|
|
9
|
+
sym = (matrix + matrix.T) / 2
|
|
10
|
+
w, v = np.linalg.eigh(sym)
|
|
11
|
+
return (v * np.clip(w, floor, None)) @ v.T
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def nearest_correlation(matrix: np.ndarray) -> np.ndarray:
|
|
15
|
+
fixed = _clip_psd(matrix)
|
|
16
|
+
d = np.sqrt(np.diag(fixed))
|
|
17
|
+
out = fixed / np.outer(d, d)
|
|
18
|
+
np.fill_diagonal(out, 1.0)
|
|
19
|
+
return out
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _cholesky(matrix: np.ndarray) -> np.ndarray:
|
|
23
|
+
jitter = 1e-10
|
|
24
|
+
for _ in range(8):
|
|
25
|
+
try:
|
|
26
|
+
return np.linalg.cholesky(matrix + jitter * np.eye(len(matrix)))
|
|
27
|
+
except np.linalg.LinAlgError:
|
|
28
|
+
jitter *= 100
|
|
29
|
+
return np.linalg.cholesky(_clip_psd(matrix, 1e-6))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class GaussianCopula:
|
|
33
|
+
"""Learns the correlation of normal scores and samples from it."""
|
|
34
|
+
|
|
35
|
+
def fit(self, Z: np.ndarray) -> GaussianCopula:
|
|
36
|
+
n, d = Z.shape
|
|
37
|
+
self.dim = d
|
|
38
|
+
if d == 0:
|
|
39
|
+
self.corr = np.zeros((0, 0))
|
|
40
|
+
elif n < 2 or d == 1:
|
|
41
|
+
self.corr = np.eye(d)
|
|
42
|
+
else:
|
|
43
|
+
with np.errstate(invalid="ignore", divide="ignore"):
|
|
44
|
+
corr = np.corrcoef(Z, rowvar=False)
|
|
45
|
+
corr = np.nan_to_num(np.atleast_2d(corr), nan=0.0)
|
|
46
|
+
np.fill_diagonal(corr, 1.0)
|
|
47
|
+
self.corr = nearest_correlation(corr)
|
|
48
|
+
self._chol = _cholesky(self.corr) if d else np.zeros((0, 0))
|
|
49
|
+
self._cond_cache: dict = {}
|
|
50
|
+
return self
|
|
51
|
+
|
|
52
|
+
def sample(self, n: int, rng: np.random.Generator) -> np.ndarray:
|
|
53
|
+
if self.dim == 0:
|
|
54
|
+
return np.zeros((n, 0))
|
|
55
|
+
return rng.standard_normal((n, self.dim)) @ self._chol.T
|
|
56
|
+
|
|
57
|
+
def sample_conditional(
|
|
58
|
+
self, given_idx: list[int], given: np.ndarray, rng: np.random.Generator
|
|
59
|
+
) -> np.ndarray:
|
|
60
|
+
"""Sample the free dimensions given the values of ``given_idx`` (one row per sample)."""
|
|
61
|
+
n = len(given)
|
|
62
|
+
if not given_idx:
|
|
63
|
+
return self.sample(n, rng)
|
|
64
|
+
key = tuple(given_idx)
|
|
65
|
+
if key not in self._cond_cache:
|
|
66
|
+
free = [i for i in range(self.dim) if i not in set(given_idx)]
|
|
67
|
+
s11 = self.corr[np.ix_(free, free)]
|
|
68
|
+
s12 = self.corr[np.ix_(free, given_idx)]
|
|
69
|
+
s22 = self.corr[np.ix_(given_idx, given_idx)]
|
|
70
|
+
a = np.linalg.solve(s22 + 1e-9 * np.eye(len(given_idx)), s12.T).T
|
|
71
|
+
cov = _clip_psd(s11 - a @ s12.T, 1e-10) if free else np.zeros((0, 0))
|
|
72
|
+
chol = _cholesky(cov) if free else cov
|
|
73
|
+
self._cond_cache[key] = (free, a, chol)
|
|
74
|
+
free, a, chol = self._cond_cache[key]
|
|
75
|
+
out = np.empty((n, self.dim))
|
|
76
|
+
out[:, given_idx] = given
|
|
77
|
+
if free:
|
|
78
|
+
out[:, free] = given @ a.T + rng.standard_normal((n, len(free))) @ chol.T
|
|
79
|
+
return out
|
synthschema/datasets.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Demo datasets. These are simulated (no real people or companies) and deterministic."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from .generators import _builtin_pii
|
|
9
|
+
from .metadata import Metadata
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _retail(n_customers: int = 1000, seed: int = 0) -> tuple[dict[str, pd.DataFrame], Metadata]:
|
|
13
|
+
rng = np.random.default_rng(seed)
|
|
14
|
+
n = n_customers
|
|
15
|
+
region = rng.choice(["North", "South", "West"], size=n, p=[0.4, 0.35, 0.25])
|
|
16
|
+
age = np.clip(np.round(rng.normal(42, 12, n)), 18, 85).astype(int)
|
|
17
|
+
seg_p = np.where(age[:, None] > 45, [0.45, 0.35, 0.20], [0.65, 0.15, 0.20])
|
|
18
|
+
segment = np.array(
|
|
19
|
+
[rng.choice(["Consumer", "Corporate", "Small Business"], p=p) for p in seg_p], dtype=object
|
|
20
|
+
)
|
|
21
|
+
seg_mult = pd.Series(segment).map({"Consumer": 1.0, "Corporate": 1.8, "Small Business": 1.4}).to_numpy()
|
|
22
|
+
income = np.round(np.exp(rng.normal(10.6, 0.35, n)) * seg_mult * (0.7 + age / 100), -2)
|
|
23
|
+
signup = pd.Timestamp("2019-01-01") + pd.to_timedelta(rng.integers(0, 6 * 365, n), unit="D")
|
|
24
|
+
first, last = _builtin_pii("first_name", n, rng), _builtin_pii("last_name", n, rng)
|
|
25
|
+
email = [f"{fn.lower()}.{ln.lower()}{i}@example.com" for i, (fn, ln) in enumerate(zip(first, last, strict=True))]
|
|
26
|
+
tenure_days = (pd.Timestamp("2025-01-01") - signup).days.to_numpy()
|
|
27
|
+
active_p = 1 / (1 + np.exp(-(1.5 - tenure_days / 1000 + (income > 60000) * 0.5)))
|
|
28
|
+
customers = pd.DataFrame({
|
|
29
|
+
"customer_id": np.arange(1, n + 1),
|
|
30
|
+
"first_name": first,
|
|
31
|
+
"last_name": last,
|
|
32
|
+
"email": email,
|
|
33
|
+
"age": age,
|
|
34
|
+
"region": region,
|
|
35
|
+
"segment": segment,
|
|
36
|
+
"income": income,
|
|
37
|
+
"signup_date": signup.strftime("%Y-%m-%d"),
|
|
38
|
+
"is_active": rng.random(n) < active_p,
|
|
39
|
+
})
|
|
40
|
+
|
|
41
|
+
lam = 1.5 + (income / income.mean()) * 1.5 + (segment == "Corporate") * 1.0
|
|
42
|
+
counts = rng.poisson(lam)
|
|
43
|
+
owner = np.repeat(np.arange(n), counts)
|
|
44
|
+
m = len(owner)
|
|
45
|
+
order_date = signup[owner] + pd.to_timedelta(rng.integers(0, 365, m), unit="D")
|
|
46
|
+
ship_days = rng.choice([1, 2, 3, 4, 5, 7], size=m, p=[0.15, 0.3, 0.25, 0.15, 0.1, 0.05])
|
|
47
|
+
channel = np.where(age[owner] < 40,
|
|
48
|
+
rng.choice(["Online", "Store", "Phone"], size=m, p=[0.7, 0.25, 0.05]),
|
|
49
|
+
rng.choice(["Online", "Store", "Phone"], size=m, p=[0.4, 0.45, 0.15]))
|
|
50
|
+
amount = np.round(np.exp(rng.normal(3.6, 0.6, m)) * (income[owner] / 50000) ** 0.6 * seg_mult[owner], 2)
|
|
51
|
+
discount = np.round(amount * rng.uniform(0, 0.3, m) * (rng.random(m) < 0.4), 2)
|
|
52
|
+
status = rng.choice(["delivered", "shipped", "returned", "cancelled"], size=m, p=[0.8, 0.1, 0.06, 0.04])
|
|
53
|
+
orders = pd.DataFrame({
|
|
54
|
+
"order_id": [f"ORD-{i:06d}" for i in range(1, m + 1)],
|
|
55
|
+
"customer_id": customers["customer_id"].to_numpy()[owner],
|
|
56
|
+
"order_date": order_date.strftime("%Y-%m-%d"),
|
|
57
|
+
"ship_date": (order_date + pd.to_timedelta(ship_days, unit="D")).strftime("%Y-%m-%d"),
|
|
58
|
+
"channel": channel,
|
|
59
|
+
"status": status,
|
|
60
|
+
"amount": amount,
|
|
61
|
+
"discount": discount,
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
md = Metadata()
|
|
65
|
+
t = md.add_table("customers", primary_key="customer_id")
|
|
66
|
+
md.add_column("customers", "customer_id", "id", representation="int")
|
|
67
|
+
md.add_column("customers", "first_name", "first_name")
|
|
68
|
+
md.add_column("customers", "last_name", "last_name")
|
|
69
|
+
md.add_column("customers", "email", "email")
|
|
70
|
+
md.add_column("customers", "age", "numerical", representation="int")
|
|
71
|
+
md.add_column("customers", "region", "categorical")
|
|
72
|
+
md.add_column("customers", "segment", "categorical")
|
|
73
|
+
md.add_column("customers", "income", "numerical")
|
|
74
|
+
md.add_column("customers", "signup_date", "datetime", datetime_format="%Y-%m-%d")
|
|
75
|
+
md.add_column("customers", "is_active", "boolean")
|
|
76
|
+
t.primary_key = "customer_id"
|
|
77
|
+
md.add_table("orders", primary_key="order_id")
|
|
78
|
+
md.add_column("orders", "order_id", "id", regex_format="ORD-[0-9]{6}")
|
|
79
|
+
md.add_column("orders", "customer_id", "id", representation="int")
|
|
80
|
+
md.add_column("orders", "order_date", "datetime", datetime_format="%Y-%m-%d")
|
|
81
|
+
md.add_column("orders", "ship_date", "datetime", datetime_format="%Y-%m-%d")
|
|
82
|
+
md.add_column("orders", "channel", "categorical")
|
|
83
|
+
md.add_column("orders", "status", "categorical")
|
|
84
|
+
md.add_column("orders", "amount", "numerical")
|
|
85
|
+
md.add_column("orders", "discount", "numerical")
|
|
86
|
+
md.add_relationship("customers", "orders", "customer_id")
|
|
87
|
+
md.add_constraint("orders", "ship_date >= order_date")
|
|
88
|
+
md.add_constraint("orders", "discount <= amount")
|
|
89
|
+
md.add_constraint("orders", "discount >= 0")
|
|
90
|
+
md.validate()
|
|
91
|
+
return {"customers": customers, "orders": orders}, md
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def load_demo(name: str = "retail", seed: int = 0):
|
|
95
|
+
"""Load a simulated demo dataset.
|
|
96
|
+
|
|
97
|
+
* ``"retail"`` - ``(dict of DataFrames, Metadata)`` with customers and orders.
|
|
98
|
+
* ``"customers"`` - ``(DataFrame, Metadata)`` with the customers table only.
|
|
99
|
+
"""
|
|
100
|
+
data, md = _retail(seed=seed)
|
|
101
|
+
if name == "retail":
|
|
102
|
+
return data, md
|
|
103
|
+
if name == "customers":
|
|
104
|
+
single = Metadata.from_dict({"tables": {"customers": md.tables["customers"].to_dict()}})
|
|
105
|
+
return data["customers"], single
|
|
106
|
+
raise ValueError(f"Unknown demo {name!r}; choose 'retail' or 'customers'")
|
|
@@ -0,0 +1,446 @@
|
|
|
1
|
+
"""Evaluation: is the synthetic data valid, faithful and private?
|
|
2
|
+
|
|
3
|
+
* :func:`evaluate_quality` - statistical fidelity (column shapes, column-pair trends,
|
|
4
|
+
and child-count shapes for multi-table data). Scores are in [0, 1]; higher is better.
|
|
5
|
+
* :func:`run_diagnostic` - structural validity: keys, foreign keys, ranges, categories, rules.
|
|
6
|
+
* :func:`evaluate_privacy` - copies of real rows and distance to closest record (DCR).
|
|
7
|
+
* :func:`certify` - combines the three into a pass/fail :class:`Certificate` with hard stops.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
from collections.abc import Mapping
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from datetime import datetime, timezone
|
|
17
|
+
from itertools import combinations
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
import numpy as np
|
|
22
|
+
import pandas as pd
|
|
23
|
+
from scipy import stats
|
|
24
|
+
|
|
25
|
+
from ._version import __version__
|
|
26
|
+
from .constraints import build_constraints
|
|
27
|
+
from .metadata import Metadata, Table
|
|
28
|
+
from .transformers import datetime_to_seconds
|
|
29
|
+
|
|
30
|
+
Data = pd.DataFrame | Mapping[str, pd.DataFrame]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _as_tables(real: Data, synthetic: Data, metadata: Metadata | None) -> tuple[dict, dict, Metadata]:
|
|
34
|
+
if isinstance(real, pd.DataFrame):
|
|
35
|
+
if metadata is None:
|
|
36
|
+
metadata = Metadata.detect_from_dataframe(real)
|
|
37
|
+
name = metadata.get_table().name
|
|
38
|
+
return {name: real}, {name: synthetic}, metadata
|
|
39
|
+
if metadata is None:
|
|
40
|
+
metadata = Metadata.detect_from_dataframes(real)
|
|
41
|
+
return dict(real), dict(synthetic), metadata
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _numeric(series: pd.Series, table: Table, col: str) -> np.ndarray:
|
|
45
|
+
meta = table.columns.get(col)
|
|
46
|
+
if meta is not None and meta.sdtype == "datetime":
|
|
47
|
+
return datetime_to_seconds(series, meta.datetime_format)
|
|
48
|
+
return pd.to_numeric(series, errors="coerce").astype(float).to_numpy()
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _is_numeric(table: Table, col: str) -> bool:
|
|
52
|
+
meta = table.columns.get(col)
|
|
53
|
+
return meta is not None and meta.sdtype in {"numerical", "datetime"}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _tvd_score(a: pd.Series, b: pd.Series) -> float:
|
|
57
|
+
pa = a.astype(object).where(a.notna(), "__null__").value_counts(normalize=True)
|
|
58
|
+
pb = b.astype(object).where(b.notna(), "__null__").value_counts(normalize=True)
|
|
59
|
+
idx = pa.index.union(pb.index)
|
|
60
|
+
return float(1 - 0.5 * np.abs(pa.reindex(idx, fill_value=0) - pb.reindex(idx, fill_value=0)).sum())
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _ks_score(a: np.ndarray, b: np.ndarray) -> float:
|
|
64
|
+
a, b = a[np.isfinite(a)], b[np.isfinite(b)]
|
|
65
|
+
if len(a) == 0 or len(b) == 0:
|
|
66
|
+
return float(len(a) == len(b))
|
|
67
|
+
return float(1 - stats.ks_2samp(a, b, method="asymp").statistic)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _binned(x: np.ndarray, edges: np.ndarray) -> pd.Series:
|
|
71
|
+
out = np.digitize(x, edges[1:-1]).astype(object)
|
|
72
|
+
out[~np.isfinite(x)] = "__null__"
|
|
73
|
+
return pd.Series(out)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclass
|
|
77
|
+
class QualityReport:
|
|
78
|
+
"""Statistical fidelity. ``score`` is the mean of the property scores."""
|
|
79
|
+
|
|
80
|
+
score: float
|
|
81
|
+
properties: dict[str, float]
|
|
82
|
+
details: dict[str, pd.DataFrame] = field(default_factory=dict)
|
|
83
|
+
|
|
84
|
+
def to_dict(self) -> dict:
|
|
85
|
+
return {"score": self.score, "properties": self.properties}
|
|
86
|
+
|
|
87
|
+
def __repr__(self) -> str:
|
|
88
|
+
props = ", ".join(f"{k}: {v:.1%}" for k, v in self.properties.items())
|
|
89
|
+
return f"QualityReport(score={self.score:.1%}; {props})"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _table_quality(real: pd.DataFrame, synth: pd.DataFrame, table: Table, keys: set[str], max_pair_columns: int):
|
|
93
|
+
cols = [c for c in table.modeled_columns if c in real.columns and c in synth.columns and c not in keys]
|
|
94
|
+
shape_rows = []
|
|
95
|
+
for col in cols:
|
|
96
|
+
if _is_numeric(table, col):
|
|
97
|
+
score = _ks_score(_numeric(real[col], table, col), _numeric(synth[col], table, col))
|
|
98
|
+
metric = "KSComplement"
|
|
99
|
+
else:
|
|
100
|
+
score = _tvd_score(real[col], synth[col])
|
|
101
|
+
metric = "TVComplement"
|
|
102
|
+
shape_rows.append({"table": table.name, "column": col, "metric": metric, "score": score})
|
|
103
|
+
|
|
104
|
+
pair_rows = []
|
|
105
|
+
pair_cols = cols[:max_pair_columns]
|
|
106
|
+
prepared = {}
|
|
107
|
+
for col in pair_cols:
|
|
108
|
+
if _is_numeric(table, col):
|
|
109
|
+
r, s = _numeric(real[col], table, col), _numeric(synth[col], table, col)
|
|
110
|
+
finite = r[np.isfinite(r)]
|
|
111
|
+
edges = np.unique(np.quantile(finite, np.linspace(0, 1, 11))) if len(finite) else np.array([0, 1])
|
|
112
|
+
prepared[col] = ("num", r, s, _binned(r, edges), _binned(s, edges))
|
|
113
|
+
else:
|
|
114
|
+
r = real[col].astype(object).where(real[col].notna(), "__null__").reset_index(drop=True)
|
|
115
|
+
s = synth[col].astype(object).where(synth[col].notna(), "__null__").reset_index(drop=True)
|
|
116
|
+
prepared[col] = ("cat", None, None, r, s)
|
|
117
|
+
for a, b in combinations(pair_cols, 2):
|
|
118
|
+
ka, ra, sa, rba, sba = prepared[a]
|
|
119
|
+
kb, rb, sb, rbb, sbb = prepared[b]
|
|
120
|
+
if ka == kb == "num":
|
|
121
|
+
rc = pd.Series(ra).corr(pd.Series(rb))
|
|
122
|
+
sc = pd.Series(sa).corr(pd.Series(sb))
|
|
123
|
+
if np.isnan(rc) or np.isnan(sc):
|
|
124
|
+
continue
|
|
125
|
+
score, metric = float(1 - abs(rc - sc) / 2), "CorrelationSimilarity"
|
|
126
|
+
else:
|
|
127
|
+
jr = (rba.astype(str) + "\x1f" + rbb.astype(str)).value_counts(normalize=True)
|
|
128
|
+
js = (sba.astype(str) + "\x1f" + sbb.astype(str)).value_counts(normalize=True)
|
|
129
|
+
idx = jr.index.union(js.index)
|
|
130
|
+
score = float(1 - 0.5 * np.abs(jr.reindex(idx, fill_value=0) - js.reindex(idx, fill_value=0)).sum())
|
|
131
|
+
metric = "ContingencySimilarity"
|
|
132
|
+
pair_rows.append({"table": table.name, "column_1": a, "column_2": b, "metric": metric, "score": score})
|
|
133
|
+
return shape_rows, pair_rows
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def evaluate_quality(
|
|
137
|
+
real: Data, synthetic: Data, metadata: Metadata | None = None, max_pair_columns: int = 40
|
|
138
|
+
) -> QualityReport:
|
|
139
|
+
"""Score how closely the synthetic data reproduces the real data's statistics."""
|
|
140
|
+
real_t, synth_t, md = _as_tables(real, synthetic, metadata)
|
|
141
|
+
shapes, pairs, cards = [], [], []
|
|
142
|
+
for name, table in md.tables.items():
|
|
143
|
+
keys = set(md.foreign_keys(name)) | ({table.primary_key} if table.primary_key else set())
|
|
144
|
+
s, p = _table_quality(real_t[name], synth_t[name], table, keys, max_pair_columns)
|
|
145
|
+
shapes.extend(s)
|
|
146
|
+
pairs.extend(p)
|
|
147
|
+
for rel in md.relationships:
|
|
148
|
+
rc = real_t[rel.child_table][rel.child_foreign_key].value_counts()
|
|
149
|
+
sc = synth_t[rel.child_table][rel.child_foreign_key].value_counts()
|
|
150
|
+
r_counts = real_t[rel.parent_table][rel.parent_primary_key].map(rc).fillna(0).to_numpy(dtype=float)
|
|
151
|
+
s_counts = synth_t[rel.parent_table][rel.parent_primary_key].map(sc).fillna(0).to_numpy(dtype=float)
|
|
152
|
+
cards.append({
|
|
153
|
+
"parent_table": rel.parent_table, "child_table": rel.child_table,
|
|
154
|
+
"metric": "CardinalityShape", "score": _ks_score(r_counts, s_counts),
|
|
155
|
+
})
|
|
156
|
+
details = {
|
|
157
|
+
"Column Shapes": pd.DataFrame(shapes),
|
|
158
|
+
"Column Pair Trends": pd.DataFrame(pairs),
|
|
159
|
+
}
|
|
160
|
+
properties = {
|
|
161
|
+
"Column Shapes": float(np.mean([r["score"] for r in shapes])) if shapes else 1.0,
|
|
162
|
+
"Column Pair Trends": float(np.mean([r["score"] for r in pairs])) if pairs else 1.0,
|
|
163
|
+
}
|
|
164
|
+
if cards:
|
|
165
|
+
details["Cardinality"] = pd.DataFrame(cards)
|
|
166
|
+
properties["Cardinality"] = float(np.mean([r["score"] for r in cards]))
|
|
167
|
+
return QualityReport(float(np.mean(list(properties.values()))), properties, details)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# ---------------------------------------------------------------------- diagnostic
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@dataclass
|
|
174
|
+
class DiagnosticReport:
|
|
175
|
+
"""Structural checks. ``passed`` is False if any hard check fails."""
|
|
176
|
+
|
|
177
|
+
checks: pd.DataFrame
|
|
178
|
+
|
|
179
|
+
@property
|
|
180
|
+
def passed(self) -> bool:
|
|
181
|
+
hard = self.checks[self.checks["hard"]]
|
|
182
|
+
return bool((hard["score"] >= 1.0).all())
|
|
183
|
+
|
|
184
|
+
@property
|
|
185
|
+
def score(self) -> float:
|
|
186
|
+
return float(self.checks["score"].mean()) if len(self.checks) else 1.0
|
|
187
|
+
|
|
188
|
+
def failures(self) -> pd.DataFrame:
|
|
189
|
+
return self.checks[self.checks["score"] < 1.0]
|
|
190
|
+
|
|
191
|
+
def to_dict(self) -> dict:
|
|
192
|
+
return {"passed": self.passed, "score": self.score, "checks": self.checks.to_dict("records")}
|
|
193
|
+
|
|
194
|
+
def __repr__(self) -> str:
|
|
195
|
+
return f"DiagnosticReport(passed={self.passed}, score={self.score:.1%}, checks={len(self.checks)})"
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def run_diagnostic(real: Data, synthetic: Data, metadata: Metadata | None = None) -> DiagnosticReport:
|
|
199
|
+
"""Check keys, foreign keys, column presence, ranges, categories and rules."""
|
|
200
|
+
real_t, synth_t, md = _as_tables(real, synthetic, metadata)
|
|
201
|
+
rows: list[dict[str, Any]] = []
|
|
202
|
+
|
|
203
|
+
def add(table, check, column, score, hard):
|
|
204
|
+
rows.append({"table": table, "check": check, "column": column, "score": float(score), "hard": hard})
|
|
205
|
+
|
|
206
|
+
for name, table in md.tables.items():
|
|
207
|
+
r, s = real_t[name], synth_t[name]
|
|
208
|
+
missing = [c for c in r.columns if c not in s.columns]
|
|
209
|
+
add(name, "columns present", None, 1.0 - len(missing) / max(1, len(r.columns)), True)
|
|
210
|
+
if table.primary_key and table.primary_key in s.columns:
|
|
211
|
+
pk = s[table.primary_key]
|
|
212
|
+
add(name, "primary key unique and non-null", table.primary_key,
|
|
213
|
+
float(pk.notna().all() and pk.is_unique), True)
|
|
214
|
+
keys = set(md.foreign_keys(name)) | ({table.primary_key} if table.primary_key else set())
|
|
215
|
+
for col in table.modeled_columns:
|
|
216
|
+
if col in keys or col not in s.columns:
|
|
217
|
+
continue
|
|
218
|
+
if _is_numeric(table, col):
|
|
219
|
+
rv, sv = _numeric(r[col], table, col), _numeric(s[col], table, col)
|
|
220
|
+
rv, sv = rv[np.isfinite(rv)], sv[np.isfinite(sv)]
|
|
221
|
+
if len(rv) and len(sv):
|
|
222
|
+
within = ((sv >= rv.min()) & (sv <= rv.max())).mean()
|
|
223
|
+
add(name, "within real min/max", col, within, False)
|
|
224
|
+
else:
|
|
225
|
+
allowed = set(r[col].dropna().unique())
|
|
226
|
+
sv = s[col].dropna()
|
|
227
|
+
if len(sv):
|
|
228
|
+
add(name, "categories seen in real data", col, sv.isin(allowed).mean(), False)
|
|
229
|
+
for constraint in build_constraints(table.constraints):
|
|
230
|
+
if all(c in s.columns for c in constraint.columns):
|
|
231
|
+
constraint.fit(r, table)
|
|
232
|
+
add(name, f"rule {constraint.to_dict()}", None, constraint.is_valid(s).mean(), True)
|
|
233
|
+
for rel in md.relationships:
|
|
234
|
+
fk = synth_t[rel.child_table][rel.child_foreign_key].dropna()
|
|
235
|
+
parent_keys = set(synth_t[rel.parent_table][rel.parent_primary_key].dropna())
|
|
236
|
+
score = fk.isin(parent_keys).mean() if len(fk) else 1.0
|
|
237
|
+
add(rel.child_table, f"foreign key -> {rel.parent_table}", rel.child_foreign_key, score, True)
|
|
238
|
+
return DiagnosticReport(pd.DataFrame(rows, columns=["table", "check", "column", "score", "hard"]))
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
# ---------------------------------------------------------------------- privacy
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
@dataclass
|
|
245
|
+
class PrivacyReport:
|
|
246
|
+
exact_match_rate: float
|
|
247
|
+
unique_match_rate: float
|
|
248
|
+
dcr_median: float | None
|
|
249
|
+
baseline_dcr_median: float | None
|
|
250
|
+
details: dict[str, Any] = field(default_factory=dict)
|
|
251
|
+
|
|
252
|
+
@property
|
|
253
|
+
def dcr_ratio(self) -> float | None:
|
|
254
|
+
if self.dcr_median is None or not self.baseline_dcr_median:
|
|
255
|
+
return None
|
|
256
|
+
return self.dcr_median / self.baseline_dcr_median
|
|
257
|
+
|
|
258
|
+
def to_dict(self) -> dict:
|
|
259
|
+
return {
|
|
260
|
+
"exact_match_rate": self.exact_match_rate,
|
|
261
|
+
"unique_match_rate": self.unique_match_rate,
|
|
262
|
+
"dcr_median": self.dcr_median,
|
|
263
|
+
"baseline_dcr_median": self.baseline_dcr_median,
|
|
264
|
+
"dcr_ratio": self.dcr_ratio,
|
|
265
|
+
**self.details,
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
def __repr__(self) -> str:
|
|
269
|
+
ratio = "n/a" if self.dcr_ratio is None else f"{self.dcr_ratio:.2f}"
|
|
270
|
+
return (
|
|
271
|
+
f"PrivacyReport(exact_match_rate={self.exact_match_rate:.2%}, "
|
|
272
|
+
f"unique_match_rate={self.unique_match_rate:.2%}, dcr_ratio={ratio})"
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _encode(df: pd.DataFrame, table: Table, cols: list[str], ref: pd.DataFrame) -> np.ndarray:
|
|
277
|
+
parts = []
|
|
278
|
+
for col in cols:
|
|
279
|
+
if _is_numeric(table, col):
|
|
280
|
+
r = _numeric(ref[col], table, col)
|
|
281
|
+
x = _numeric(df[col], table, col)
|
|
282
|
+
mu, sd = np.nanmean(r), np.nanstd(r) or 1.0
|
|
283
|
+
parts.append(np.nan_to_num((x - mu) / sd)[:, None])
|
|
284
|
+
else:
|
|
285
|
+
top = ref[col].astype(str).value_counts().index[:20]
|
|
286
|
+
vals = df[col].astype(str).to_numpy()
|
|
287
|
+
parts.append(np.stack([(vals == t).astype(float) for t in top], axis=1) if len(top) else np.zeros((len(df), 0)))
|
|
288
|
+
return np.hstack(parts) if parts else np.zeros((len(df), 0))
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _min_dist(a: np.ndarray, b: np.ndarray, exclude_self: bool = False) -> np.ndarray:
|
|
292
|
+
out = np.empty(len(a))
|
|
293
|
+
b2 = (b**2).sum(axis=1)
|
|
294
|
+
for start in range(0, len(a), 512):
|
|
295
|
+
chunk = a[start:start + 512]
|
|
296
|
+
d = (chunk**2).sum(axis=1)[:, None] + b2[None, :] - 2 * chunk @ b.T
|
|
297
|
+
d = np.sqrt(np.clip(d, 0, None))
|
|
298
|
+
if exclude_self:
|
|
299
|
+
idx = np.arange(start, start + len(chunk))
|
|
300
|
+
d[np.arange(len(chunk)), idx] = np.inf
|
|
301
|
+
out[start:start + len(chunk)] = d.min(axis=1)
|
|
302
|
+
return out
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def evaluate_privacy(
|
|
306
|
+
real: pd.DataFrame,
|
|
307
|
+
synthetic: pd.DataFrame,
|
|
308
|
+
metadata: Metadata | None = None,
|
|
309
|
+
table_name: str | None = None,
|
|
310
|
+
max_rows: int = 2000,
|
|
311
|
+
seed: int = 0,
|
|
312
|
+
) -> PrivacyReport:
|
|
313
|
+
"""Measure copying of real rows (single table).
|
|
314
|
+
|
|
315
|
+
* ``exact_match_rate`` - share of synthetic rows identical to some real row (modelled columns).
|
|
316
|
+
* ``unique_match_rate`` - share of synthetic rows identical to a real row that is *unique*
|
|
317
|
+
in the real data. This is the re-identification risk; it should be 0.
|
|
318
|
+
* DCR - median distance from synthetic rows to their closest real row, compared with the
|
|
319
|
+
same distance between real rows (``dcr_ratio`` near or above 1 means no memorisation).
|
|
320
|
+
"""
|
|
321
|
+
if metadata is None:
|
|
322
|
+
metadata = Metadata.detect_from_dataframe(real)
|
|
323
|
+
table = metadata.get_table(table_name)
|
|
324
|
+
keys = {table.primary_key} | set(metadata.foreign_keys(table.name))
|
|
325
|
+
cols = [c for c in table.modeled_columns if c in synthetic.columns and c not in keys]
|
|
326
|
+
if not cols:
|
|
327
|
+
return PrivacyReport(0.0, 0.0, None, None, {"columns": []})
|
|
328
|
+
r_hash = pd.util.hash_pandas_object(real[cols].astype(str), index=False)
|
|
329
|
+
s_hash = pd.util.hash_pandas_object(synthetic[cols].astype(str), index=False)
|
|
330
|
+
counts = r_hash.value_counts()
|
|
331
|
+
matched = s_hash.isin(counts.index)
|
|
332
|
+
unique_hashes = set(counts[counts == 1].index)
|
|
333
|
+
exact = float(matched.mean()) if len(s_hash) else 0.0
|
|
334
|
+
unique = float(s_hash.isin(unique_hashes).mean()) if len(s_hash) else 0.0
|
|
335
|
+
|
|
336
|
+
rng = np.random.default_rng(seed)
|
|
337
|
+
r_idx = rng.choice(len(real), min(len(real), max_rows * 2), replace=False)
|
|
338
|
+
s_idx = rng.choice(len(synthetic), min(len(synthetic), max_rows), replace=False)
|
|
339
|
+
ref = real.iloc[r_idx].reset_index(drop=True)
|
|
340
|
+
R = _encode(ref, table, cols, real)
|
|
341
|
+
S = _encode(synthetic.iloc[s_idx].reset_index(drop=True), table, cols, real)
|
|
342
|
+
dcr = baseline = None
|
|
343
|
+
if R.shape[1] and len(R) > 1 and len(S):
|
|
344
|
+
dcr = float(np.median(_min_dist(S, R)))
|
|
345
|
+
half = min(len(R), max_rows)
|
|
346
|
+
baseline = float(np.median(_min_dist(R[:half], R, exclude_self=True)))
|
|
347
|
+
return PrivacyReport(exact, unique, dcr, baseline, {"columns": cols})
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
# ---------------------------------------------------------------------- certificate
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
@dataclass
|
|
354
|
+
class Certificate:
|
|
355
|
+
"""Pass/fail release decision with the evidence behind it."""
|
|
356
|
+
|
|
357
|
+
passed: bool
|
|
358
|
+
reasons: list[str]
|
|
359
|
+
quality: QualityReport
|
|
360
|
+
diagnostic: DiagnosticReport
|
|
361
|
+
privacy: dict[str, PrivacyReport]
|
|
362
|
+
thresholds: dict[str, float]
|
|
363
|
+
fingerprint: str
|
|
364
|
+
created_at: str
|
|
365
|
+
version: str = __version__
|
|
366
|
+
|
|
367
|
+
def to_dict(self) -> dict:
|
|
368
|
+
return {
|
|
369
|
+
"passed": self.passed,
|
|
370
|
+
"reasons": self.reasons,
|
|
371
|
+
"quality": self.quality.to_dict(),
|
|
372
|
+
"diagnostic": {"passed": self.diagnostic.passed, "score": self.diagnostic.score,
|
|
373
|
+
"failures": self.diagnostic.failures().to_dict("records")},
|
|
374
|
+
"privacy": {k: v.to_dict() for k, v in self.privacy.items()},
|
|
375
|
+
"thresholds": self.thresholds,
|
|
376
|
+
"fingerprint": self.fingerprint,
|
|
377
|
+
"created_at": self.created_at,
|
|
378
|
+
"synthschema_version": self.version,
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
def to_json(self, indent: int = 2) -> str:
|
|
382
|
+
return json.dumps(self.to_dict(), indent=indent, default=str)
|
|
383
|
+
|
|
384
|
+
def save(self, path: str | Path) -> None:
|
|
385
|
+
Path(path).write_text(self.to_json())
|
|
386
|
+
|
|
387
|
+
def __repr__(self) -> str:
|
|
388
|
+
status = "PASSED" if self.passed else "FAILED"
|
|
389
|
+
return (
|
|
390
|
+
f"Certificate({status}; quality={self.quality.score:.1%}, "
|
|
391
|
+
f"diagnostic={'ok' if self.diagnostic.passed else 'failed'}, reasons={self.reasons})"
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _fingerprint(synth_t: Mapping[str, pd.DataFrame], md: Metadata) -> str:
|
|
396
|
+
h = hashlib.sha256()
|
|
397
|
+
h.update(json.dumps(md.to_dict(), sort_keys=True, default=str).encode())
|
|
398
|
+
for name in sorted(synth_t):
|
|
399
|
+
h.update(name.encode())
|
|
400
|
+
h.update(pd.util.hash_pandas_object(synth_t[name].astype(str), index=False).to_numpy().tobytes())
|
|
401
|
+
return h.hexdigest()
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def certify(
|
|
405
|
+
real: Data,
|
|
406
|
+
synthetic: Data,
|
|
407
|
+
metadata: Metadata | None = None,
|
|
408
|
+
*,
|
|
409
|
+
min_quality: float = 0.80,
|
|
410
|
+
max_unique_match_rate: float = 0.0,
|
|
411
|
+
min_dcr_ratio: float | None = None,
|
|
412
|
+
) -> Certificate:
|
|
413
|
+
"""Decide whether synthetic data may be released.
|
|
414
|
+
|
|
415
|
+
Hard stops: any failed structural check (keys, foreign keys, rules), any copy of a
|
|
416
|
+
real row that is unique in the real data above ``max_unique_match_rate``, or quality
|
|
417
|
+
below ``min_quality``.
|
|
418
|
+
"""
|
|
419
|
+
real_t, synth_t, md = _as_tables(real, synthetic, metadata)
|
|
420
|
+
quality = evaluate_quality(real_t, synth_t, md)
|
|
421
|
+
diagnostic = run_diagnostic(real_t, synth_t, md)
|
|
422
|
+
privacy = {name: evaluate_privacy(real_t[name], synth_t[name], md, name) for name in md.tables}
|
|
423
|
+
reasons = []
|
|
424
|
+
if not diagnostic.passed:
|
|
425
|
+
for row in diagnostic.failures().query("hard").to_dict("records"):
|
|
426
|
+
reasons.append(f"structure: {row['table']}: {row['check']} ({row['score']:.1%})")
|
|
427
|
+
if quality.score < min_quality:
|
|
428
|
+
reasons.append(f"quality {quality.score:.1%} is below {min_quality:.0%}")
|
|
429
|
+
for name, rep in privacy.items():
|
|
430
|
+
if rep.unique_match_rate > max_unique_match_rate:
|
|
431
|
+
reasons.append(f"privacy: {name}: {rep.unique_match_rate:.2%} of rows copy a unique real row")
|
|
432
|
+
if min_dcr_ratio is not None and rep.dcr_ratio is not None and rep.dcr_ratio < min_dcr_ratio:
|
|
433
|
+
reasons.append(f"privacy: {name}: DCR ratio {rep.dcr_ratio:.2f} is below {min_dcr_ratio}")
|
|
434
|
+
thresholds = {"min_quality": min_quality, "max_unique_match_rate": max_unique_match_rate}
|
|
435
|
+
if min_dcr_ratio is not None:
|
|
436
|
+
thresholds["min_dcr_ratio"] = min_dcr_ratio
|
|
437
|
+
return Certificate(
|
|
438
|
+
passed=not reasons,
|
|
439
|
+
reasons=reasons,
|
|
440
|
+
quality=quality,
|
|
441
|
+
diagnostic=diagnostic,
|
|
442
|
+
privacy=privacy,
|
|
443
|
+
thresholds=thresholds,
|
|
444
|
+
fingerprint=_fingerprint(synth_t, md),
|
|
445
|
+
created_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
446
|
+
)
|