synthschema 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,547 @@
1
+ """Metadata: the blueprint that describes tables, columns, keys, relationships and rules.
2
+
3
+ One metadata object drives everything in synthschema: profiling, generation, constraints and
4
+ evaluation all read the same description, so a rule is declared once.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import re
11
+ from collections import deque
12
+ from collections.abc import Iterable, Mapping
13
+ from dataclasses import dataclass, field, fields
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ import pandas as pd
18
+
19
+ #: Column types that are learned by the statistical model.
20
+ MODELED_SDTYPES = frozenset({"numerical", "categorical", "boolean", "datetime"})
21
+
22
+ #: Personally identifiable column types. These are never learned; values are always
23
+ #: generated from scratch (with Faker when it is installed).
24
+ PII_SDTYPES = frozenset(
25
+ {
26
+ "email",
27
+ "phone_number",
28
+ "name",
29
+ "first_name",
30
+ "last_name",
31
+ "address",
32
+ "street_address",
33
+ "ssn",
34
+ "credit_card_number",
35
+ "ip_address",
36
+ "url",
37
+ "company",
38
+ "postcode",
39
+ }
40
+ )
41
+
42
+ #: Column types that are generated rather than learned.
43
+ GENERATED_SDTYPES = frozenset({"id", "text"}) | PII_SDTYPES
44
+
45
+ ALL_SDTYPES = MODELED_SDTYPES | GENERATED_SDTYPES
46
+
47
+
48
+ @dataclass
49
+ class Column:
50
+ """Description of one column.
51
+
52
+ Only ``sdtype`` is required. The other fields are optional hints. Learned synthesizers
53
+ use the data itself; the schema-only synthesizer relies on these hints.
54
+ """
55
+
56
+ sdtype: str
57
+ datetime_format: str | None = None
58
+ regex_format: str | None = None
59
+ representation: str | None = None # "int", "float", "date" or "datetime"
60
+ values: list | None = None
61
+ weights: list | None = None
62
+ min_value: Any = None
63
+ max_value: Any = None
64
+ decimals: int | None = None
65
+ null_rate: float | None = None
66
+ unique: bool | None = None
67
+ description: str | None = None
68
+
69
+ def __post_init__(self) -> None:
70
+ if self.sdtype not in ALL_SDTYPES:
71
+ raise ValueError(
72
+ f"Unknown sdtype {self.sdtype!r}. Expected one of: {', '.join(sorted(ALL_SDTYPES))}"
73
+ )
74
+
75
+ @property
76
+ def is_pii(self) -> bool:
77
+ return self.sdtype in PII_SDTYPES
78
+
79
+ @property
80
+ def is_modeled(self) -> bool:
81
+ return self.sdtype in MODELED_SDTYPES
82
+
83
+ def to_dict(self) -> dict:
84
+ out = {}
85
+ for f in fields(self):
86
+ value = getattr(self, f.name)
87
+ if value is not None:
88
+ out[f.name] = value
89
+ return out
90
+
91
+ @classmethod
92
+ def from_dict(cls, data: Mapping | str) -> Column:
93
+ if isinstance(data, str):
94
+ return cls(sdtype=data)
95
+ known = {f.name for f in fields(cls)}
96
+ unknown = set(data) - known
97
+ if unknown:
98
+ raise ValueError(f"Unknown column field(s): {', '.join(sorted(unknown))}")
99
+ return cls(**dict(data))
100
+
101
+
102
+ @dataclass
103
+ class Relationship:
104
+ """A one-to-many link: each child row points at one parent row (or is null)."""
105
+
106
+ parent_table: str
107
+ parent_primary_key: str
108
+ child_table: str
109
+ child_foreign_key: str
110
+
111
+ def to_dict(self) -> dict:
112
+ return {
113
+ "parent_table": self.parent_table,
114
+ "parent_primary_key": self.parent_primary_key,
115
+ "child_table": self.child_table,
116
+ "child_foreign_key": self.child_foreign_key,
117
+ }
118
+
119
+
120
+ @dataclass
121
+ class Table:
122
+ name: str
123
+ columns: dict[str, Column] = field(default_factory=dict)
124
+ primary_key: str | None = None
125
+ constraints: list = field(default_factory=list)
126
+
127
+ def to_dict(self) -> dict:
128
+ out: dict[str, Any] = {"columns": {n: c.to_dict() for n, c in self.columns.items()}}
129
+ if self.primary_key:
130
+ out["primary_key"] = self.primary_key
131
+ if self.constraints:
132
+ out["constraints"] = [
133
+ c.to_dict() if hasattr(c, "to_dict") else c for c in self.constraints
134
+ ]
135
+ return out
136
+
137
+ @property
138
+ def modeled_columns(self) -> list[str]:
139
+ return [n for n, c in self.columns.items() if c.is_modeled]
140
+
141
+
142
+ class Metadata:
143
+ """Tables, columns, keys, relationships and rules for one dataset.
144
+
145
+ Build it by hand, detect it from data with :meth:`detect_from_dataframe` /
146
+ :meth:`detect_from_dataframes`, parse it from SQL DDL with :meth:`from_ddl`, or load a
147
+ JSON/YAML blueprint with :meth:`load`.
148
+ """
149
+
150
+ def __init__(self) -> None:
151
+ self.tables: dict[str, Table] = {}
152
+ self.relationships: list[Relationship] = []
153
+
154
+ # ------------------------------------------------------------------ building
155
+ def add_table(self, name: str, primary_key: str | None = None) -> Table:
156
+ if name in self.tables:
157
+ raise ValueError(f"Table {name!r} already exists")
158
+ table = Table(name=name, primary_key=primary_key)
159
+ self.tables[name] = table
160
+ return table
161
+
162
+ def add_column(self, table: str, name: str, sdtype: str, **hints: Any) -> Column:
163
+ column = Column(sdtype=sdtype, **hints)
164
+ self.get_table(table).columns[name] = column
165
+ return column
166
+
167
+ def update_column(self, table: str, name: str, **changes: Any) -> Column:
168
+ t = self.get_table(table)
169
+ if name not in t.columns:
170
+ raise KeyError(f"Column {name!r} not found in table {table!r}")
171
+ data = t.columns[name].to_dict()
172
+ data.update(changes)
173
+ t.columns[name] = Column.from_dict(data)
174
+ return t.columns[name]
175
+
176
+ def set_primary_key(self, table: str, column: str) -> None:
177
+ t = self.get_table(table)
178
+ if column not in t.columns:
179
+ raise KeyError(f"Column {column!r} not found in table {table!r}")
180
+ if t.columns[column].sdtype != "id":
181
+ self.update_column(table, column, sdtype="id")
182
+ t.primary_key = column
183
+
184
+ def add_relationship(
185
+ self,
186
+ parent_table: str,
187
+ child_table: str,
188
+ child_foreign_key: str,
189
+ parent_primary_key: str | None = None,
190
+ ) -> Relationship:
191
+ parent = self.get_table(parent_table)
192
+ pk = parent_primary_key or parent.primary_key
193
+ if pk is None:
194
+ raise ValueError(f"Parent table {parent_table!r} has no primary key")
195
+ rel = Relationship(parent_table, pk, child_table, child_foreign_key)
196
+ self.relationships.append(rel)
197
+ child = self.get_table(child_table)
198
+ if child_foreign_key in child.columns and child.columns[child_foreign_key].sdtype != "id":
199
+ self.update_column(child_table, child_foreign_key, sdtype="id")
200
+ return rel
201
+
202
+ def add_constraint(self, table: str, constraint: Any) -> None:
203
+ """Attach a rule to a table: a dict, a rule string such as ``"ship_date >= order_date"``,
204
+ or a :class:`synthschema.constraints.Constraint` object."""
205
+ self.get_table(table).constraints.append(constraint)
206
+
207
+ # ------------------------------------------------------------------ access
208
+ def get_table(self, name: str | None = None) -> Table:
209
+ if name is None:
210
+ if len(self.tables) != 1:
211
+ raise ValueError("Metadata has several tables; pass a table name")
212
+ return next(iter(self.tables.values()))
213
+ if name not in self.tables:
214
+ raise KeyError(f"Table {name!r} not found. Tables: {list(self.tables)}")
215
+ return self.tables[name]
216
+
217
+ @property
218
+ def table_names(self) -> list[str]:
219
+ return list(self.tables)
220
+
221
+ def foreign_keys(self, table: str) -> list[str]:
222
+ return [r.child_foreign_key for r in self.relationships if r.child_table == table]
223
+
224
+ def parent_relationships(self, table: str) -> list[Relationship]:
225
+ return [
226
+ r for r in self.relationships if r.child_table == table and r.parent_table != table
227
+ ]
228
+
229
+ def child_relationships(self, table: str) -> list[Relationship]:
230
+ return [
231
+ r for r in self.relationships if r.parent_table == table and r.child_table != table
232
+ ]
233
+
234
+ def topological_order(self) -> list[str]:
235
+ """Tables ordered so that every parent comes before its children."""
236
+ indegree = {t: 0 for t in self.tables}
237
+ children: dict[str, list[str]] = {t: [] for t in self.tables}
238
+ for r in self.relationships:
239
+ if r.parent_table == r.child_table:
240
+ continue
241
+ indegree[r.child_table] += 1
242
+ children[r.parent_table].append(r.child_table)
243
+ queue = deque(t for t in self.tables if indegree[t] == 0)
244
+ order: list[str] = []
245
+ while queue:
246
+ t = queue.popleft()
247
+ order.append(t)
248
+ for c in children[t]:
249
+ indegree[c] -= 1
250
+ if indegree[c] == 0:
251
+ queue.append(c)
252
+ if len(order) != len(self.tables):
253
+ raise ValueError("Relationships contain a cycle; synthschema needs a parent-first order")
254
+ return order
255
+
256
+ # ------------------------------------------------------------------ validation
257
+ def validate(self) -> None:
258
+ """Raise ``ValueError`` if keys or relationships are inconsistent."""
259
+ errors = []
260
+ for t in self.tables.values():
261
+ if t.primary_key is not None:
262
+ if t.primary_key not in t.columns:
263
+ errors.append(f"{t.name}: primary key {t.primary_key!r} is not a column")
264
+ elif t.columns[t.primary_key].sdtype != "id":
265
+ errors.append(f"{t.name}: primary key {t.primary_key!r} must have sdtype 'id'")
266
+ for r in self.relationships:
267
+ if r.parent_table not in self.tables:
268
+ errors.append(f"relationship: unknown parent table {r.parent_table!r}")
269
+ continue
270
+ if r.child_table not in self.tables:
271
+ errors.append(f"relationship: unknown child table {r.child_table!r}")
272
+ continue
273
+ parent = self.tables[r.parent_table]
274
+ if parent.primary_key != r.parent_primary_key:
275
+ errors.append(
276
+ f"relationship {r.parent_table}->{r.child_table}: "
277
+ f"{r.parent_primary_key!r} is not the primary key of {r.parent_table!r}"
278
+ )
279
+ if r.child_foreign_key not in self.tables[r.child_table].columns:
280
+ errors.append(
281
+ f"relationship {r.parent_table}->{r.child_table}: "
282
+ f"foreign key {r.child_foreign_key!r} is not a column"
283
+ )
284
+ if errors:
285
+ raise ValueError("Invalid metadata:\n " + "\n ".join(errors))
286
+ self.topological_order()
287
+
288
+ def validate_data(self, data: pd.DataFrame | Mapping[str, pd.DataFrame]) -> None:
289
+ if isinstance(data, pd.DataFrame):
290
+ data = {self.get_table().name: data}
291
+ errors = []
292
+ for name, table in self.tables.items():
293
+ if name not in data:
294
+ errors.append(f"missing data for table {name!r}")
295
+ continue
296
+ missing = [c for c in table.columns if c not in data[name].columns]
297
+ if missing:
298
+ errors.append(f"{name}: columns missing from data: {missing}")
299
+ if errors:
300
+ raise ValueError("Data does not match metadata:\n " + "\n ".join(errors))
301
+
302
+ # ------------------------------------------------------------------ detection
303
+ @classmethod
304
+ def detect_from_dataframe(cls, data: pd.DataFrame, table_name: str = "table") -> Metadata:
305
+ """Infer column types and the primary key of one table."""
306
+ md = cls()
307
+ md._add_detected_table(table_name, data)
308
+ return md
309
+
310
+ @classmethod
311
+ def detect_from_dataframes(cls, data: Mapping[str, pd.DataFrame]) -> Metadata:
312
+ """Infer tables, keys and foreign-key relationships from several tables."""
313
+ md = cls()
314
+ for name, df in data.items():
315
+ md._add_detected_table(name, df)
316
+ for child_name, child_df in data.items():
317
+ child = md.tables[child_name]
318
+ for col in child_df.columns:
319
+ if col == child.primary_key:
320
+ continue
321
+ for parent_name, parent in md.tables.items():
322
+ if parent_name == child_name or parent.primary_key is None:
323
+ continue
324
+ if not _fk_name_matches(col, parent_name, parent.primary_key):
325
+ continue
326
+ child_vals = child_df[col].dropna()
327
+ if child_vals.empty:
328
+ continue
329
+ parent_keys = set(data[parent_name][parent.primary_key].dropna())
330
+ if child_vals.isin(parent_keys).mean() >= 0.9:
331
+ md.add_relationship(parent_name, child_name, col)
332
+ break
333
+ return md
334
+
335
+ def _add_detected_table(self, name: str, df: pd.DataFrame) -> None:
336
+ table = self.add_table(name)
337
+ for col in df.columns:
338
+ table.columns[str(col)] = detect_column(df[col], str(col))
339
+ table.primary_key = _detect_primary_key(name, df, table)
340
+
341
+ # ------------------------------------------------------------------ serialisation
342
+ def to_dict(self) -> dict:
343
+ return {
344
+ "tables": {n: t.to_dict() for n, t in self.tables.items()},
345
+ "relationships": [r.to_dict() for r in self.relationships],
346
+ }
347
+
348
+ @classmethod
349
+ def from_dict(cls, data: Mapping) -> Metadata:
350
+ md = cls()
351
+ for name, tdata in data.get("tables", {}).items():
352
+ table = md.add_table(name, primary_key=tdata.get("primary_key"))
353
+ for cname, cdata in tdata.get("columns", {}).items():
354
+ table.columns[cname] = Column.from_dict(cdata)
355
+ table.constraints = list(tdata.get("constraints", []))
356
+ for rdata in data.get("relationships", []):
357
+ md.relationships.append(Relationship(**rdata))
358
+ md.validate()
359
+ return md
360
+
361
+ def save(self, path: str | Path) -> None:
362
+ """Write the blueprint as JSON, or YAML if the path ends in .yaml/.yml."""
363
+ path = Path(path)
364
+ if path.suffix.lower() in {".yaml", ".yml"}:
365
+ yaml = _import_yaml()
366
+ path.write_text(yaml.safe_dump(self.to_dict(), sort_keys=False, default_flow_style=False))
367
+ else:
368
+ path.write_text(json.dumps(self.to_dict(), indent=2, default=str))
369
+
370
+ @classmethod
371
+ def load(cls, path: str | Path) -> Metadata:
372
+ path = Path(path)
373
+ text = path.read_text()
374
+ if path.suffix.lower() in {".yaml", ".yml"}:
375
+ return cls.from_dict(_import_yaml().safe_load(text))
376
+ return cls.from_dict(json.loads(text))
377
+
378
+ @classmethod
379
+ def from_ddl(cls, ddl: str | Path) -> Metadata:
380
+ """Build metadata from SQL ``CREATE TABLE`` statements (a string or a .sql path)."""
381
+ from .schema import parse_ddl
382
+
383
+ return parse_ddl(ddl)
384
+
385
+ def __repr__(self) -> str:
386
+ parts = []
387
+ for t in self.tables.values():
388
+ parts.append(f"{t.name}({len(t.columns)} columns, pk={t.primary_key})")
389
+ return f"Metadata(tables=[{', '.join(parts)}], relationships={len(self.relationships)})"
390
+
391
+
392
+ # ---------------------------------------------------------------------- detection helpers
393
+
394
+ _DATE_FORMATS = [
395
+ "%Y-%m-%d",
396
+ "%Y-%m-%d %H:%M:%S",
397
+ "%Y-%m-%dT%H:%M:%S",
398
+ "%Y-%m-%d %H:%M",
399
+ "%m/%d/%Y",
400
+ "%d/%m/%Y",
401
+ "%Y/%m/%d",
402
+ "%d-%m-%Y",
403
+ "%m/%d/%Y %H:%M:%S",
404
+ "%m/%d/%Y %H:%M",
405
+ ]
406
+
407
+ _EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[A-Za-z]{2,}$")
408
+
409
+ _PII_NAME_RULES: list[tuple[re.Pattern, str]] = [
410
+ (re.compile(r"e[-_ ]?mail"), "email"),
411
+ (re.compile(r"(phone|mobile|cell[-_ ]?no|fax|telephone|^tel$)"), "phone_number"),
412
+ (re.compile(r"^(first|given)[-_ ]?name$|^fname$"), "first_name"),
413
+ (re.compile(r"^(last|family)[-_ ]?name$|^surname$|^lname$"), "last_name"),
414
+ (re.compile(r"^(full[-_ ]?)?name$|^(customer|person|employee|contact|patient|client)[-_ ]?name$"), "name"),
415
+ (re.compile(r"(street|address|addr)"), "address"),
416
+ (re.compile(r"(^ssn$|social[-_ ]?security|national[-_ ]?id|^nin$)"), "ssn"),
417
+ (re.compile(r"(credit[-_ ]?card|card[-_ ]?number|^ccn$|^pan$)"), "credit_card_number"),
418
+ (re.compile(r"(ip[-_ ]?address|^ip$|^ipv4$)"), "ip_address"),
419
+ (re.compile(r"(^url$|website|homepage)"), "url"),
420
+ ]
421
+
422
+
423
+ def _is_id_name(name: str) -> bool:
424
+ lname = name.lower()
425
+ return (
426
+ lname in {"id", "uuid", "guid", "key", "pk"}
427
+ or lname.endswith(("_id", "_key", "_uuid", "_guid", "_pk"))
428
+ or (name.endswith("Id") or name.endswith("ID"))
429
+ )
430
+
431
+
432
+ def _fk_name_matches(col: str, parent_table: str, parent_pk: str) -> bool:
433
+ lcol, lparent, lpk = col.lower(), parent_table.lower(), parent_pk.lower()
434
+ if lcol == lpk and lpk != "id":
435
+ return True
436
+ stems = {lparent, lparent.rstrip("s"), lparent[:-3] + "y" if lparent.endswith("ies") else lparent}
437
+ return any(lcol in (f"{s}_id", f"{s}id", f"{s}_{lpk}") for s in stems)
438
+
439
+
440
+ def _is_integer_valued(values: pd.Series) -> bool:
441
+ if pd.api.types.is_integer_dtype(values):
442
+ return True
443
+ try:
444
+ arr = values.astype(float).to_numpy()
445
+ except (TypeError, ValueError):
446
+ return False
447
+ return bool(len(arr) > 0 and (arr == arr.round()).all())
448
+
449
+
450
+ def detect_datetime_format(values: pd.Series) -> str | None | bool:
451
+ """Return a strptime format that parses every value, ``None`` for ISO-8601 strings that
452
+ need no explicit format, or ``False`` if the values are not dates."""
453
+ sample = values.astype(str).head(200)
454
+ if not sample.str.contains(r"\d", regex=True).all():
455
+ return False
456
+ for fmt in _DATE_FORMATS:
457
+ parsed = pd.to_datetime(sample, format=fmt, errors="coerce")
458
+ if parsed.notna().all():
459
+ return fmt
460
+ if sample.str.match(r"^\d{4}-\d{2}-\d{2}[T ]").all():
461
+ parsed = pd.to_datetime(sample, format="ISO8601", errors="coerce")
462
+ if parsed.notna().all():
463
+ return None
464
+ return False
465
+
466
+
467
+ def detect_column(series: pd.Series, name: str) -> Column:
468
+ """Infer the :class:`Column` description of one pandas Series."""
469
+ from .generators import infer_regex
470
+
471
+ nonnull = series.dropna()
472
+ n, nn = len(series), len(nonnull)
473
+ if nn == 0:
474
+ return Column("categorical")
475
+ nunique = nonnull.nunique()
476
+ is_unique = nunique == nn == n
477
+
478
+ if pd.api.types.is_bool_dtype(series):
479
+ return Column("boolean")
480
+ if pd.api.types.is_datetime64_any_dtype(series):
481
+ return Column("datetime")
482
+ if pd.api.types.is_numeric_dtype(series):
483
+ integer = _is_integer_valued(nonnull)
484
+ lname = name.lower()
485
+ if _is_id_name(name) and integer:
486
+ if is_unique or nunique > 0.5 * nn:
487
+ return Column("id", representation="int")
488
+ return Column("categorical")
489
+ for pattern, sdtype in _PII_NAME_RULES:
490
+ if sdtype in {"phone_number", "ssn", "credit_card_number"} and pattern.search(lname):
491
+ return Column(sdtype)
492
+ return Column("numerical", representation="int" if integer else "float")
493
+
494
+ strings = nonnull.astype(str)
495
+ sample = strings.sample(min(nn, 500), random_state=0) if nn > 500 else strings
496
+ if sample.str.match(_EMAIL_RE).mean() >= 0.9:
497
+ return Column("email")
498
+ lname = name.lower()
499
+ for pattern, sdtype in _PII_NAME_RULES:
500
+ if pattern.search(lname):
501
+ return Column(sdtype)
502
+
503
+ fmt = detect_datetime_format(sample)
504
+ if fmt is not False:
505
+ return Column("datetime", datetime_format=fmt)
506
+
507
+ ratio = nunique / nn
508
+ if _is_id_name(name) and (is_unique or ratio > 0.5):
509
+ return Column("id", regex_format=infer_regex(strings))
510
+ avg_len = sample.str.len().mean()
511
+ space_share = sample.str.contains(" ").mean()
512
+ if ratio > 0.5 and nunique > 20 and avg_len >= 20 and space_share > 0.5:
513
+ return Column("text")
514
+ if ratio > 0.9 and nunique > 50:
515
+ if space_share > 0.3:
516
+ return Column("text")
517
+ return Column("id", regex_format=infer_regex(strings))
518
+ return Column("categorical")
519
+
520
+
521
+ def _detect_primary_key(table_name: str, df: pd.DataFrame, table: Table) -> str | None:
522
+ candidates = []
523
+ for col, meta in table.columns.items():
524
+ if meta.sdtype != "id":
525
+ continue
526
+ s = df[col]
527
+ if s.isna().any() or not s.is_unique:
528
+ continue
529
+ lcol = col.lower()
530
+ stem = table_name.lower().rstrip("s")
531
+ rank = 0 if lcol in {"id", f"{stem}_id", f"{table_name.lower()}_id", f"{stem}id"} else 1
532
+ candidates.append((rank, list(df.columns).index(col), col))
533
+ if not candidates:
534
+ return None
535
+ return sorted(candidates)[0][2]
536
+
537
+
538
+ def _import_yaml():
539
+ try:
540
+ import yaml
541
+ except ImportError as exc: # pragma: no cover - depends on environment
542
+ raise ImportError("YAML blueprints need PyYAML: pip install 'synthschema[yaml]'") from exc
543
+ return yaml
544
+
545
+
546
+ def as_metadata_dict(tables: Iterable[Table]) -> dict:
547
+ return {t.name: t.to_dict() for t in tables}