synthschema 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synthschema/__init__.py +56 -0
- synthschema/_version.py +1 -0
- synthschema/cli.py +134 -0
- synthschema/constraints.py +438 -0
- synthschema/copula.py +79 -0
- synthschema/datasets.py +106 -0
- synthschema/evaluation.py +446 -0
- synthschema/generators.py +456 -0
- synthschema/marginals.py +143 -0
- synthschema/metadata.py +547 -0
- synthschema/multi_table.py +272 -0
- synthschema/py.typed +0 -0
- synthschema/schema.py +490 -0
- synthschema/single_table.py +311 -0
- synthschema/transformers.py +247 -0
- synthschema-0.1.0.dist-info/METADATA +253 -0
- synthschema-0.1.0.dist-info/RECORD +20 -0
- synthschema-0.1.0.dist-info/WHEEL +4 -0
- synthschema-0.1.0.dist-info/entry_points.txt +2 -0
- synthschema-0.1.0.dist-info/licenses/LICENSE +201 -0
synthschema/metadata.py
ADDED
|
@@ -0,0 +1,547 @@
|
|
|
1
|
+
"""Metadata: the blueprint that describes tables, columns, keys, relationships and rules.
|
|
2
|
+
|
|
3
|
+
One metadata object drives everything in synthschema: profiling, generation, constraints and
|
|
4
|
+
evaluation all read the same description, so a rule is declared once.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
from collections import deque
|
|
12
|
+
from collections.abc import Iterable, Mapping
|
|
13
|
+
from dataclasses import dataclass, field, fields
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
import pandas as pd
|
|
18
|
+
|
|
19
|
+
#: Column types that are learned by the statistical model.
|
|
20
|
+
MODELED_SDTYPES = frozenset({"numerical", "categorical", "boolean", "datetime"})
|
|
21
|
+
|
|
22
|
+
#: Personally identifiable column types. These are never learned; values are always
|
|
23
|
+
#: generated from scratch (with Faker when it is installed).
|
|
24
|
+
PII_SDTYPES = frozenset(
|
|
25
|
+
{
|
|
26
|
+
"email",
|
|
27
|
+
"phone_number",
|
|
28
|
+
"name",
|
|
29
|
+
"first_name",
|
|
30
|
+
"last_name",
|
|
31
|
+
"address",
|
|
32
|
+
"street_address",
|
|
33
|
+
"ssn",
|
|
34
|
+
"credit_card_number",
|
|
35
|
+
"ip_address",
|
|
36
|
+
"url",
|
|
37
|
+
"company",
|
|
38
|
+
"postcode",
|
|
39
|
+
}
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
#: Column types that are generated rather than learned.
|
|
43
|
+
GENERATED_SDTYPES = frozenset({"id", "text"}) | PII_SDTYPES
|
|
44
|
+
|
|
45
|
+
ALL_SDTYPES = MODELED_SDTYPES | GENERATED_SDTYPES
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class Column:
|
|
50
|
+
"""Description of one column.
|
|
51
|
+
|
|
52
|
+
Only ``sdtype`` is required. The other fields are optional hints. Learned synthesizers
|
|
53
|
+
use the data itself; the schema-only synthesizer relies on these hints.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
sdtype: str
|
|
57
|
+
datetime_format: str | None = None
|
|
58
|
+
regex_format: str | None = None
|
|
59
|
+
representation: str | None = None # "int", "float", "date" or "datetime"
|
|
60
|
+
values: list | None = None
|
|
61
|
+
weights: list | None = None
|
|
62
|
+
min_value: Any = None
|
|
63
|
+
max_value: Any = None
|
|
64
|
+
decimals: int | None = None
|
|
65
|
+
null_rate: float | None = None
|
|
66
|
+
unique: bool | None = None
|
|
67
|
+
description: str | None = None
|
|
68
|
+
|
|
69
|
+
def __post_init__(self) -> None:
|
|
70
|
+
if self.sdtype not in ALL_SDTYPES:
|
|
71
|
+
raise ValueError(
|
|
72
|
+
f"Unknown sdtype {self.sdtype!r}. Expected one of: {', '.join(sorted(ALL_SDTYPES))}"
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def is_pii(self) -> bool:
|
|
77
|
+
return self.sdtype in PII_SDTYPES
|
|
78
|
+
|
|
79
|
+
@property
|
|
80
|
+
def is_modeled(self) -> bool:
|
|
81
|
+
return self.sdtype in MODELED_SDTYPES
|
|
82
|
+
|
|
83
|
+
def to_dict(self) -> dict:
|
|
84
|
+
out = {}
|
|
85
|
+
for f in fields(self):
|
|
86
|
+
value = getattr(self, f.name)
|
|
87
|
+
if value is not None:
|
|
88
|
+
out[f.name] = value
|
|
89
|
+
return out
|
|
90
|
+
|
|
91
|
+
@classmethod
|
|
92
|
+
def from_dict(cls, data: Mapping | str) -> Column:
|
|
93
|
+
if isinstance(data, str):
|
|
94
|
+
return cls(sdtype=data)
|
|
95
|
+
known = {f.name for f in fields(cls)}
|
|
96
|
+
unknown = set(data) - known
|
|
97
|
+
if unknown:
|
|
98
|
+
raise ValueError(f"Unknown column field(s): {', '.join(sorted(unknown))}")
|
|
99
|
+
return cls(**dict(data))
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass
|
|
103
|
+
class Relationship:
|
|
104
|
+
"""A one-to-many link: each child row points at one parent row (or is null)."""
|
|
105
|
+
|
|
106
|
+
parent_table: str
|
|
107
|
+
parent_primary_key: str
|
|
108
|
+
child_table: str
|
|
109
|
+
child_foreign_key: str
|
|
110
|
+
|
|
111
|
+
def to_dict(self) -> dict:
|
|
112
|
+
return {
|
|
113
|
+
"parent_table": self.parent_table,
|
|
114
|
+
"parent_primary_key": self.parent_primary_key,
|
|
115
|
+
"child_table": self.child_table,
|
|
116
|
+
"child_foreign_key": self.child_foreign_key,
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@dataclass
|
|
121
|
+
class Table:
|
|
122
|
+
name: str
|
|
123
|
+
columns: dict[str, Column] = field(default_factory=dict)
|
|
124
|
+
primary_key: str | None = None
|
|
125
|
+
constraints: list = field(default_factory=list)
|
|
126
|
+
|
|
127
|
+
def to_dict(self) -> dict:
|
|
128
|
+
out: dict[str, Any] = {"columns": {n: c.to_dict() for n, c in self.columns.items()}}
|
|
129
|
+
if self.primary_key:
|
|
130
|
+
out["primary_key"] = self.primary_key
|
|
131
|
+
if self.constraints:
|
|
132
|
+
out["constraints"] = [
|
|
133
|
+
c.to_dict() if hasattr(c, "to_dict") else c for c in self.constraints
|
|
134
|
+
]
|
|
135
|
+
return out
|
|
136
|
+
|
|
137
|
+
@property
|
|
138
|
+
def modeled_columns(self) -> list[str]:
|
|
139
|
+
return [n for n, c in self.columns.items() if c.is_modeled]
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class Metadata:
|
|
143
|
+
"""Tables, columns, keys, relationships and rules for one dataset.
|
|
144
|
+
|
|
145
|
+
Build it by hand, detect it from data with :meth:`detect_from_dataframe` /
|
|
146
|
+
:meth:`detect_from_dataframes`, parse it from SQL DDL with :meth:`from_ddl`, or load a
|
|
147
|
+
JSON/YAML blueprint with :meth:`load`.
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
def __init__(self) -> None:
|
|
151
|
+
self.tables: dict[str, Table] = {}
|
|
152
|
+
self.relationships: list[Relationship] = []
|
|
153
|
+
|
|
154
|
+
# ------------------------------------------------------------------ building
|
|
155
|
+
def add_table(self, name: str, primary_key: str | None = None) -> Table:
|
|
156
|
+
if name in self.tables:
|
|
157
|
+
raise ValueError(f"Table {name!r} already exists")
|
|
158
|
+
table = Table(name=name, primary_key=primary_key)
|
|
159
|
+
self.tables[name] = table
|
|
160
|
+
return table
|
|
161
|
+
|
|
162
|
+
def add_column(self, table: str, name: str, sdtype: str, **hints: Any) -> Column:
|
|
163
|
+
column = Column(sdtype=sdtype, **hints)
|
|
164
|
+
self.get_table(table).columns[name] = column
|
|
165
|
+
return column
|
|
166
|
+
|
|
167
|
+
def update_column(self, table: str, name: str, **changes: Any) -> Column:
|
|
168
|
+
t = self.get_table(table)
|
|
169
|
+
if name not in t.columns:
|
|
170
|
+
raise KeyError(f"Column {name!r} not found in table {table!r}")
|
|
171
|
+
data = t.columns[name].to_dict()
|
|
172
|
+
data.update(changes)
|
|
173
|
+
t.columns[name] = Column.from_dict(data)
|
|
174
|
+
return t.columns[name]
|
|
175
|
+
|
|
176
|
+
def set_primary_key(self, table: str, column: str) -> None:
|
|
177
|
+
t = self.get_table(table)
|
|
178
|
+
if column not in t.columns:
|
|
179
|
+
raise KeyError(f"Column {column!r} not found in table {table!r}")
|
|
180
|
+
if t.columns[column].sdtype != "id":
|
|
181
|
+
self.update_column(table, column, sdtype="id")
|
|
182
|
+
t.primary_key = column
|
|
183
|
+
|
|
184
|
+
def add_relationship(
|
|
185
|
+
self,
|
|
186
|
+
parent_table: str,
|
|
187
|
+
child_table: str,
|
|
188
|
+
child_foreign_key: str,
|
|
189
|
+
parent_primary_key: str | None = None,
|
|
190
|
+
) -> Relationship:
|
|
191
|
+
parent = self.get_table(parent_table)
|
|
192
|
+
pk = parent_primary_key or parent.primary_key
|
|
193
|
+
if pk is None:
|
|
194
|
+
raise ValueError(f"Parent table {parent_table!r} has no primary key")
|
|
195
|
+
rel = Relationship(parent_table, pk, child_table, child_foreign_key)
|
|
196
|
+
self.relationships.append(rel)
|
|
197
|
+
child = self.get_table(child_table)
|
|
198
|
+
if child_foreign_key in child.columns and child.columns[child_foreign_key].sdtype != "id":
|
|
199
|
+
self.update_column(child_table, child_foreign_key, sdtype="id")
|
|
200
|
+
return rel
|
|
201
|
+
|
|
202
|
+
def add_constraint(self, table: str, constraint: Any) -> None:
|
|
203
|
+
"""Attach a rule to a table: a dict, a rule string such as ``"ship_date >= order_date"``,
|
|
204
|
+
or a :class:`synthschema.constraints.Constraint` object."""
|
|
205
|
+
self.get_table(table).constraints.append(constraint)
|
|
206
|
+
|
|
207
|
+
# ------------------------------------------------------------------ access
|
|
208
|
+
def get_table(self, name: str | None = None) -> Table:
|
|
209
|
+
if name is None:
|
|
210
|
+
if len(self.tables) != 1:
|
|
211
|
+
raise ValueError("Metadata has several tables; pass a table name")
|
|
212
|
+
return next(iter(self.tables.values()))
|
|
213
|
+
if name not in self.tables:
|
|
214
|
+
raise KeyError(f"Table {name!r} not found. Tables: {list(self.tables)}")
|
|
215
|
+
return self.tables[name]
|
|
216
|
+
|
|
217
|
+
@property
|
|
218
|
+
def table_names(self) -> list[str]:
|
|
219
|
+
return list(self.tables)
|
|
220
|
+
|
|
221
|
+
def foreign_keys(self, table: str) -> list[str]:
|
|
222
|
+
return [r.child_foreign_key for r in self.relationships if r.child_table == table]
|
|
223
|
+
|
|
224
|
+
def parent_relationships(self, table: str) -> list[Relationship]:
|
|
225
|
+
return [
|
|
226
|
+
r for r in self.relationships if r.child_table == table and r.parent_table != table
|
|
227
|
+
]
|
|
228
|
+
|
|
229
|
+
def child_relationships(self, table: str) -> list[Relationship]:
|
|
230
|
+
return [
|
|
231
|
+
r for r in self.relationships if r.parent_table == table and r.child_table != table
|
|
232
|
+
]
|
|
233
|
+
|
|
234
|
+
def topological_order(self) -> list[str]:
|
|
235
|
+
"""Tables ordered so that every parent comes before its children."""
|
|
236
|
+
indegree = {t: 0 for t in self.tables}
|
|
237
|
+
children: dict[str, list[str]] = {t: [] for t in self.tables}
|
|
238
|
+
for r in self.relationships:
|
|
239
|
+
if r.parent_table == r.child_table:
|
|
240
|
+
continue
|
|
241
|
+
indegree[r.child_table] += 1
|
|
242
|
+
children[r.parent_table].append(r.child_table)
|
|
243
|
+
queue = deque(t for t in self.tables if indegree[t] == 0)
|
|
244
|
+
order: list[str] = []
|
|
245
|
+
while queue:
|
|
246
|
+
t = queue.popleft()
|
|
247
|
+
order.append(t)
|
|
248
|
+
for c in children[t]:
|
|
249
|
+
indegree[c] -= 1
|
|
250
|
+
if indegree[c] == 0:
|
|
251
|
+
queue.append(c)
|
|
252
|
+
if len(order) != len(self.tables):
|
|
253
|
+
raise ValueError("Relationships contain a cycle; synthschema needs a parent-first order")
|
|
254
|
+
return order
|
|
255
|
+
|
|
256
|
+
# ------------------------------------------------------------------ validation
|
|
257
|
+
def validate(self) -> None:
|
|
258
|
+
"""Raise ``ValueError`` if keys or relationships are inconsistent."""
|
|
259
|
+
errors = []
|
|
260
|
+
for t in self.tables.values():
|
|
261
|
+
if t.primary_key is not None:
|
|
262
|
+
if t.primary_key not in t.columns:
|
|
263
|
+
errors.append(f"{t.name}: primary key {t.primary_key!r} is not a column")
|
|
264
|
+
elif t.columns[t.primary_key].sdtype != "id":
|
|
265
|
+
errors.append(f"{t.name}: primary key {t.primary_key!r} must have sdtype 'id'")
|
|
266
|
+
for r in self.relationships:
|
|
267
|
+
if r.parent_table not in self.tables:
|
|
268
|
+
errors.append(f"relationship: unknown parent table {r.parent_table!r}")
|
|
269
|
+
continue
|
|
270
|
+
if r.child_table not in self.tables:
|
|
271
|
+
errors.append(f"relationship: unknown child table {r.child_table!r}")
|
|
272
|
+
continue
|
|
273
|
+
parent = self.tables[r.parent_table]
|
|
274
|
+
if parent.primary_key != r.parent_primary_key:
|
|
275
|
+
errors.append(
|
|
276
|
+
f"relationship {r.parent_table}->{r.child_table}: "
|
|
277
|
+
f"{r.parent_primary_key!r} is not the primary key of {r.parent_table!r}"
|
|
278
|
+
)
|
|
279
|
+
if r.child_foreign_key not in self.tables[r.child_table].columns:
|
|
280
|
+
errors.append(
|
|
281
|
+
f"relationship {r.parent_table}->{r.child_table}: "
|
|
282
|
+
f"foreign key {r.child_foreign_key!r} is not a column"
|
|
283
|
+
)
|
|
284
|
+
if errors:
|
|
285
|
+
raise ValueError("Invalid metadata:\n " + "\n ".join(errors))
|
|
286
|
+
self.topological_order()
|
|
287
|
+
|
|
288
|
+
def validate_data(self, data: pd.DataFrame | Mapping[str, pd.DataFrame]) -> None:
|
|
289
|
+
if isinstance(data, pd.DataFrame):
|
|
290
|
+
data = {self.get_table().name: data}
|
|
291
|
+
errors = []
|
|
292
|
+
for name, table in self.tables.items():
|
|
293
|
+
if name not in data:
|
|
294
|
+
errors.append(f"missing data for table {name!r}")
|
|
295
|
+
continue
|
|
296
|
+
missing = [c for c in table.columns if c not in data[name].columns]
|
|
297
|
+
if missing:
|
|
298
|
+
errors.append(f"{name}: columns missing from data: {missing}")
|
|
299
|
+
if errors:
|
|
300
|
+
raise ValueError("Data does not match metadata:\n " + "\n ".join(errors))
|
|
301
|
+
|
|
302
|
+
# ------------------------------------------------------------------ detection
|
|
303
|
+
@classmethod
|
|
304
|
+
def detect_from_dataframe(cls, data: pd.DataFrame, table_name: str = "table") -> Metadata:
|
|
305
|
+
"""Infer column types and the primary key of one table."""
|
|
306
|
+
md = cls()
|
|
307
|
+
md._add_detected_table(table_name, data)
|
|
308
|
+
return md
|
|
309
|
+
|
|
310
|
+
@classmethod
|
|
311
|
+
def detect_from_dataframes(cls, data: Mapping[str, pd.DataFrame]) -> Metadata:
|
|
312
|
+
"""Infer tables, keys and foreign-key relationships from several tables."""
|
|
313
|
+
md = cls()
|
|
314
|
+
for name, df in data.items():
|
|
315
|
+
md._add_detected_table(name, df)
|
|
316
|
+
for child_name, child_df in data.items():
|
|
317
|
+
child = md.tables[child_name]
|
|
318
|
+
for col in child_df.columns:
|
|
319
|
+
if col == child.primary_key:
|
|
320
|
+
continue
|
|
321
|
+
for parent_name, parent in md.tables.items():
|
|
322
|
+
if parent_name == child_name or parent.primary_key is None:
|
|
323
|
+
continue
|
|
324
|
+
if not _fk_name_matches(col, parent_name, parent.primary_key):
|
|
325
|
+
continue
|
|
326
|
+
child_vals = child_df[col].dropna()
|
|
327
|
+
if child_vals.empty:
|
|
328
|
+
continue
|
|
329
|
+
parent_keys = set(data[parent_name][parent.primary_key].dropna())
|
|
330
|
+
if child_vals.isin(parent_keys).mean() >= 0.9:
|
|
331
|
+
md.add_relationship(parent_name, child_name, col)
|
|
332
|
+
break
|
|
333
|
+
return md
|
|
334
|
+
|
|
335
|
+
def _add_detected_table(self, name: str, df: pd.DataFrame) -> None:
|
|
336
|
+
table = self.add_table(name)
|
|
337
|
+
for col in df.columns:
|
|
338
|
+
table.columns[str(col)] = detect_column(df[col], str(col))
|
|
339
|
+
table.primary_key = _detect_primary_key(name, df, table)
|
|
340
|
+
|
|
341
|
+
# ------------------------------------------------------------------ serialisation
|
|
342
|
+
def to_dict(self) -> dict:
|
|
343
|
+
return {
|
|
344
|
+
"tables": {n: t.to_dict() for n, t in self.tables.items()},
|
|
345
|
+
"relationships": [r.to_dict() for r in self.relationships],
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
@classmethod
|
|
349
|
+
def from_dict(cls, data: Mapping) -> Metadata:
|
|
350
|
+
md = cls()
|
|
351
|
+
for name, tdata in data.get("tables", {}).items():
|
|
352
|
+
table = md.add_table(name, primary_key=tdata.get("primary_key"))
|
|
353
|
+
for cname, cdata in tdata.get("columns", {}).items():
|
|
354
|
+
table.columns[cname] = Column.from_dict(cdata)
|
|
355
|
+
table.constraints = list(tdata.get("constraints", []))
|
|
356
|
+
for rdata in data.get("relationships", []):
|
|
357
|
+
md.relationships.append(Relationship(**rdata))
|
|
358
|
+
md.validate()
|
|
359
|
+
return md
|
|
360
|
+
|
|
361
|
+
def save(self, path: str | Path) -> None:
|
|
362
|
+
"""Write the blueprint as JSON, or YAML if the path ends in .yaml/.yml."""
|
|
363
|
+
path = Path(path)
|
|
364
|
+
if path.suffix.lower() in {".yaml", ".yml"}:
|
|
365
|
+
yaml = _import_yaml()
|
|
366
|
+
path.write_text(yaml.safe_dump(self.to_dict(), sort_keys=False, default_flow_style=False))
|
|
367
|
+
else:
|
|
368
|
+
path.write_text(json.dumps(self.to_dict(), indent=2, default=str))
|
|
369
|
+
|
|
370
|
+
@classmethod
|
|
371
|
+
def load(cls, path: str | Path) -> Metadata:
|
|
372
|
+
path = Path(path)
|
|
373
|
+
text = path.read_text()
|
|
374
|
+
if path.suffix.lower() in {".yaml", ".yml"}:
|
|
375
|
+
return cls.from_dict(_import_yaml().safe_load(text))
|
|
376
|
+
return cls.from_dict(json.loads(text))
|
|
377
|
+
|
|
378
|
+
@classmethod
|
|
379
|
+
def from_ddl(cls, ddl: str | Path) -> Metadata:
|
|
380
|
+
"""Build metadata from SQL ``CREATE TABLE`` statements (a string or a .sql path)."""
|
|
381
|
+
from .schema import parse_ddl
|
|
382
|
+
|
|
383
|
+
return parse_ddl(ddl)
|
|
384
|
+
|
|
385
|
+
def __repr__(self) -> str:
|
|
386
|
+
parts = []
|
|
387
|
+
for t in self.tables.values():
|
|
388
|
+
parts.append(f"{t.name}({len(t.columns)} columns, pk={t.primary_key})")
|
|
389
|
+
return f"Metadata(tables=[{', '.join(parts)}], relationships={len(self.relationships)})"
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
# ---------------------------------------------------------------------- detection helpers
|
|
393
|
+
|
|
394
|
+
_DATE_FORMATS = [
|
|
395
|
+
"%Y-%m-%d",
|
|
396
|
+
"%Y-%m-%d %H:%M:%S",
|
|
397
|
+
"%Y-%m-%dT%H:%M:%S",
|
|
398
|
+
"%Y-%m-%d %H:%M",
|
|
399
|
+
"%m/%d/%Y",
|
|
400
|
+
"%d/%m/%Y",
|
|
401
|
+
"%Y/%m/%d",
|
|
402
|
+
"%d-%m-%Y",
|
|
403
|
+
"%m/%d/%Y %H:%M:%S",
|
|
404
|
+
"%m/%d/%Y %H:%M",
|
|
405
|
+
]
|
|
406
|
+
|
|
407
|
+
_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[A-Za-z]{2,}$")
|
|
408
|
+
|
|
409
|
+
_PII_NAME_RULES: list[tuple[re.Pattern, str]] = [
|
|
410
|
+
(re.compile(r"e[-_ ]?mail"), "email"),
|
|
411
|
+
(re.compile(r"(phone|mobile|cell[-_ ]?no|fax|telephone|^tel$)"), "phone_number"),
|
|
412
|
+
(re.compile(r"^(first|given)[-_ ]?name$|^fname$"), "first_name"),
|
|
413
|
+
(re.compile(r"^(last|family)[-_ ]?name$|^surname$|^lname$"), "last_name"),
|
|
414
|
+
(re.compile(r"^(full[-_ ]?)?name$|^(customer|person|employee|contact|patient|client)[-_ ]?name$"), "name"),
|
|
415
|
+
(re.compile(r"(street|address|addr)"), "address"),
|
|
416
|
+
(re.compile(r"(^ssn$|social[-_ ]?security|national[-_ ]?id|^nin$)"), "ssn"),
|
|
417
|
+
(re.compile(r"(credit[-_ ]?card|card[-_ ]?number|^ccn$|^pan$)"), "credit_card_number"),
|
|
418
|
+
(re.compile(r"(ip[-_ ]?address|^ip$|^ipv4$)"), "ip_address"),
|
|
419
|
+
(re.compile(r"(^url$|website|homepage)"), "url"),
|
|
420
|
+
]
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def _is_id_name(name: str) -> bool:
|
|
424
|
+
lname = name.lower()
|
|
425
|
+
return (
|
|
426
|
+
lname in {"id", "uuid", "guid", "key", "pk"}
|
|
427
|
+
or lname.endswith(("_id", "_key", "_uuid", "_guid", "_pk"))
|
|
428
|
+
or (name.endswith("Id") or name.endswith("ID"))
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _fk_name_matches(col: str, parent_table: str, parent_pk: str) -> bool:
|
|
433
|
+
lcol, lparent, lpk = col.lower(), parent_table.lower(), parent_pk.lower()
|
|
434
|
+
if lcol == lpk and lpk != "id":
|
|
435
|
+
return True
|
|
436
|
+
stems = {lparent, lparent.rstrip("s"), lparent[:-3] + "y" if lparent.endswith("ies") else lparent}
|
|
437
|
+
return any(lcol in (f"{s}_id", f"{s}id", f"{s}_{lpk}") for s in stems)
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _is_integer_valued(values: pd.Series) -> bool:
|
|
441
|
+
if pd.api.types.is_integer_dtype(values):
|
|
442
|
+
return True
|
|
443
|
+
try:
|
|
444
|
+
arr = values.astype(float).to_numpy()
|
|
445
|
+
except (TypeError, ValueError):
|
|
446
|
+
return False
|
|
447
|
+
return bool(len(arr) > 0 and (arr == arr.round()).all())
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def detect_datetime_format(values: pd.Series) -> str | None | bool:
|
|
451
|
+
"""Return a strptime format that parses every value, ``None`` for ISO-8601 strings that
|
|
452
|
+
need no explicit format, or ``False`` if the values are not dates."""
|
|
453
|
+
sample = values.astype(str).head(200)
|
|
454
|
+
if not sample.str.contains(r"\d", regex=True).all():
|
|
455
|
+
return False
|
|
456
|
+
for fmt in _DATE_FORMATS:
|
|
457
|
+
parsed = pd.to_datetime(sample, format=fmt, errors="coerce")
|
|
458
|
+
if parsed.notna().all():
|
|
459
|
+
return fmt
|
|
460
|
+
if sample.str.match(r"^\d{4}-\d{2}-\d{2}[T ]").all():
|
|
461
|
+
parsed = pd.to_datetime(sample, format="ISO8601", errors="coerce")
|
|
462
|
+
if parsed.notna().all():
|
|
463
|
+
return None
|
|
464
|
+
return False
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def detect_column(series: pd.Series, name: str) -> Column:
|
|
468
|
+
"""Infer the :class:`Column` description of one pandas Series."""
|
|
469
|
+
from .generators import infer_regex
|
|
470
|
+
|
|
471
|
+
nonnull = series.dropna()
|
|
472
|
+
n, nn = len(series), len(nonnull)
|
|
473
|
+
if nn == 0:
|
|
474
|
+
return Column("categorical")
|
|
475
|
+
nunique = nonnull.nunique()
|
|
476
|
+
is_unique = nunique == nn == n
|
|
477
|
+
|
|
478
|
+
if pd.api.types.is_bool_dtype(series):
|
|
479
|
+
return Column("boolean")
|
|
480
|
+
if pd.api.types.is_datetime64_any_dtype(series):
|
|
481
|
+
return Column("datetime")
|
|
482
|
+
if pd.api.types.is_numeric_dtype(series):
|
|
483
|
+
integer = _is_integer_valued(nonnull)
|
|
484
|
+
lname = name.lower()
|
|
485
|
+
if _is_id_name(name) and integer:
|
|
486
|
+
if is_unique or nunique > 0.5 * nn:
|
|
487
|
+
return Column("id", representation="int")
|
|
488
|
+
return Column("categorical")
|
|
489
|
+
for pattern, sdtype in _PII_NAME_RULES:
|
|
490
|
+
if sdtype in {"phone_number", "ssn", "credit_card_number"} and pattern.search(lname):
|
|
491
|
+
return Column(sdtype)
|
|
492
|
+
return Column("numerical", representation="int" if integer else "float")
|
|
493
|
+
|
|
494
|
+
strings = nonnull.astype(str)
|
|
495
|
+
sample = strings.sample(min(nn, 500), random_state=0) if nn > 500 else strings
|
|
496
|
+
if sample.str.match(_EMAIL_RE).mean() >= 0.9:
|
|
497
|
+
return Column("email")
|
|
498
|
+
lname = name.lower()
|
|
499
|
+
for pattern, sdtype in _PII_NAME_RULES:
|
|
500
|
+
if pattern.search(lname):
|
|
501
|
+
return Column(sdtype)
|
|
502
|
+
|
|
503
|
+
fmt = detect_datetime_format(sample)
|
|
504
|
+
if fmt is not False:
|
|
505
|
+
return Column("datetime", datetime_format=fmt)
|
|
506
|
+
|
|
507
|
+
ratio = nunique / nn
|
|
508
|
+
if _is_id_name(name) and (is_unique or ratio > 0.5):
|
|
509
|
+
return Column("id", regex_format=infer_regex(strings))
|
|
510
|
+
avg_len = sample.str.len().mean()
|
|
511
|
+
space_share = sample.str.contains(" ").mean()
|
|
512
|
+
if ratio > 0.5 and nunique > 20 and avg_len >= 20 and space_share > 0.5:
|
|
513
|
+
return Column("text")
|
|
514
|
+
if ratio > 0.9 and nunique > 50:
|
|
515
|
+
if space_share > 0.3:
|
|
516
|
+
return Column("text")
|
|
517
|
+
return Column("id", regex_format=infer_regex(strings))
|
|
518
|
+
return Column("categorical")
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def _detect_primary_key(table_name: str, df: pd.DataFrame, table: Table) -> str | None:
|
|
522
|
+
candidates = []
|
|
523
|
+
for col, meta in table.columns.items():
|
|
524
|
+
if meta.sdtype != "id":
|
|
525
|
+
continue
|
|
526
|
+
s = df[col]
|
|
527
|
+
if s.isna().any() or not s.is_unique:
|
|
528
|
+
continue
|
|
529
|
+
lcol = col.lower()
|
|
530
|
+
stem = table_name.lower().rstrip("s")
|
|
531
|
+
rank = 0 if lcol in {"id", f"{stem}_id", f"{table_name.lower()}_id", f"{stem}id"} else 1
|
|
532
|
+
candidates.append((rank, list(df.columns).index(col), col))
|
|
533
|
+
if not candidates:
|
|
534
|
+
return None
|
|
535
|
+
return sorted(candidates)[0][2]
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def _import_yaml():
|
|
539
|
+
try:
|
|
540
|
+
import yaml
|
|
541
|
+
except ImportError as exc: # pragma: no cover - depends on environment
|
|
542
|
+
raise ImportError("YAML blueprints need PyYAML: pip install 'synthschema[yaml]'") from exc
|
|
543
|
+
return yaml
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def as_metadata_dict(tables: Iterable[Table]) -> dict:
|
|
547
|
+
return {t.name: t.to_dict() for t in tables}
|