synthschema 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synthschema/__init__.py +56 -0
- synthschema/_version.py +1 -0
- synthschema/cli.py +134 -0
- synthschema/constraints.py +438 -0
- synthschema/copula.py +79 -0
- synthschema/datasets.py +106 -0
- synthschema/evaluation.py +446 -0
- synthschema/generators.py +456 -0
- synthschema/marginals.py +143 -0
- synthschema/metadata.py +547 -0
- synthschema/multi_table.py +272 -0
- synthschema/py.typed +0 -0
- synthschema/schema.py +490 -0
- synthschema/single_table.py +311 -0
- synthschema/transformers.py +247 -0
- synthschema-0.1.0.dist-info/METADATA +253 -0
- synthschema-0.1.0.dist-info/RECORD +20 -0
- synthschema-0.1.0.dist-info/WHEEL +4 -0
- synthschema-0.1.0.dist-info/entry_points.txt +2 -0
- synthschema-0.1.0.dist-info/licenses/LICENSE +201 -0
synthschema/schema.py
ADDED
|
@@ -0,0 +1,490 @@
|
|
|
1
|
+
"""Schema-only generation: realistic, key-consistent data from DDL and rules, with no seed data.
|
|
2
|
+
|
|
3
|
+
Example::
|
|
4
|
+
|
|
5
|
+
from synthschema import SchemaSynthesizer
|
|
6
|
+
|
|
7
|
+
ddl = '''
|
|
8
|
+
CREATE TABLE customers (
|
|
9
|
+
customer_id INT PRIMARY KEY,
|
|
10
|
+
email VARCHAR(120) NOT NULL,
|
|
11
|
+
segment VARCHAR(20) CHECK (segment IN ('Consumer', 'Corporate')),
|
|
12
|
+
age INT CHECK (age BETWEEN 18 AND 90)
|
|
13
|
+
);
|
|
14
|
+
CREATE TABLE orders (
|
|
15
|
+
order_id INT PRIMARY KEY,
|
|
16
|
+
customer_id INT REFERENCES customers(customer_id),
|
|
17
|
+
order_date DATE,
|
|
18
|
+
ship_date DATE,
|
|
19
|
+
amount DECIMAL(10, 2) CHECK (amount >= 0),
|
|
20
|
+
CHECK (ship_date >= order_date)
|
|
21
|
+
);
|
|
22
|
+
'''
|
|
23
|
+
data = SchemaSynthesizer.from_ddl(ddl).sample(num_rows=100, seed=1)
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import re
|
|
29
|
+
from collections.abc import Mapping
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
from typing import Any
|
|
32
|
+
|
|
33
|
+
import numpy as np
|
|
34
|
+
import pandas as pd
|
|
35
|
+
|
|
36
|
+
from .constraints import Constraint, Inequality, OneOf, Range, build_constraints, parse_rule
|
|
37
|
+
from .generators import TextGenerator, generate_from_regex, generate_pii
|
|
38
|
+
from .metadata import _PII_NAME_RULES, Column, Metadata, Table
|
|
39
|
+
|
|
40
|
+
# ---------------------------------------------------------------------- DDL parsing
|
|
41
|
+
|
|
42
|
+
_CREATE_RE = re.compile(
|
|
43
|
+
r"CREATE\s+(?:OR\s+REPLACE\s+)?(?:(?:GLOBAL\s+|LOCAL\s+)?TEMP(?:ORARY)?\s+)?TABLE\s+"
|
|
44
|
+
r"(?:IF\s+NOT\s+EXISTS\s+)?([\w\.\"`\[\]]+)\s*\(",
|
|
45
|
+
re.IGNORECASE,
|
|
46
|
+
)
|
|
47
|
+
_TYPE_RE = re.compile(
|
|
48
|
+
r"^([A-Za-z_][\w]*(?:\s+(?:varying|precision|without\s+time\s+zone|with\s+time\s+zone))?)"
|
|
49
|
+
r"\s*(?:\(([^)]*)\))?",
|
|
50
|
+
re.IGNORECASE,
|
|
51
|
+
)
|
|
52
|
+
_REF_RE = re.compile(r"REFERENCES\s+([\w\.\"`\[\]]+)\s*(?:\(\s*([\w\"`\[\]]+)\s*\))?", re.IGNORECASE)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _strip_comments(sql: str) -> str:
|
|
56
|
+
sql = re.sub(r"/\*.*?\*/", " ", sql, flags=re.DOTALL)
|
|
57
|
+
return re.sub(r"--[^\n]*", " ", sql)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _ident(name: str) -> str:
|
|
61
|
+
name = name.strip().split(".")[-1]
|
|
62
|
+
return name.strip('"`[]')
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _balanced(text: str, start: int) -> tuple[str, int]:
|
|
66
|
+
"""Return the text inside the parenthesis that opens at ``start`` and the end index."""
|
|
67
|
+
depth, i, quote = 0, start, None
|
|
68
|
+
while i < len(text):
|
|
69
|
+
ch = text[i]
|
|
70
|
+
if quote:
|
|
71
|
+
if ch == quote:
|
|
72
|
+
quote = None
|
|
73
|
+
elif ch in "'\"":
|
|
74
|
+
quote = ch
|
|
75
|
+
elif ch == "(":
|
|
76
|
+
depth += 1
|
|
77
|
+
elif ch == ")":
|
|
78
|
+
depth -= 1
|
|
79
|
+
if depth == 0:
|
|
80
|
+
return text[start + 1 : i], i
|
|
81
|
+
i += 1
|
|
82
|
+
raise ValueError("Unbalanced parentheses in DDL")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _split_top(text: str, sep: str = ",") -> list[str]:
|
|
86
|
+
parts, depth, quote, cur = [], 0, None, []
|
|
87
|
+
for ch in text:
|
|
88
|
+
if quote:
|
|
89
|
+
if ch == quote:
|
|
90
|
+
quote = None
|
|
91
|
+
elif ch in "'\"":
|
|
92
|
+
quote = ch
|
|
93
|
+
elif ch == "(":
|
|
94
|
+
depth += 1
|
|
95
|
+
elif ch == ")":
|
|
96
|
+
depth -= 1
|
|
97
|
+
elif ch == sep and depth == 0:
|
|
98
|
+
parts.append("".join(cur).strip())
|
|
99
|
+
cur = []
|
|
100
|
+
continue
|
|
101
|
+
cur.append(ch)
|
|
102
|
+
if "".join(cur).strip():
|
|
103
|
+
parts.append("".join(cur).strip())
|
|
104
|
+
return parts
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _checks(text: str) -> list[str]:
|
|
108
|
+
out = []
|
|
109
|
+
for m in re.finditer(r"CHECK\s*\(", text, re.IGNORECASE):
|
|
110
|
+
inner, _ = _balanced(text, m.end() - 1)
|
|
111
|
+
out.append(inner)
|
|
112
|
+
return out
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _quoted_values(text: str) -> list[str]:
|
|
116
|
+
return [m.group(1) for m in re.finditer(r"'((?:[^']|'')*)'", text)]
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _column_from_type(name: str, sql_type: str, args: str | None) -> Column:
|
|
120
|
+
t = sql_type.lower()
|
|
121
|
+
lname = name.lower()
|
|
122
|
+
if t in {"enum", "set"}:
|
|
123
|
+
return Column("categorical", values=_quoted_values(args or ""))
|
|
124
|
+
if t.startswith(("bool", "bit")):
|
|
125
|
+
return Column("boolean")
|
|
126
|
+
if t in {"date"}:
|
|
127
|
+
return Column("datetime", representation="date")
|
|
128
|
+
if t.startswith(("timestamp", "datetime", "smalldatetime", "datetime2", "datetimeoffset")):
|
|
129
|
+
return Column("datetime", representation="datetime")
|
|
130
|
+
if t.startswith(("time",)):
|
|
131
|
+
return Column("datetime", representation="datetime")
|
|
132
|
+
if t in {"int", "integer", "bigint", "smallint", "tinyint", "mediumint", "serial", "bigserial",
|
|
133
|
+
"smallserial", "int2", "int4", "int8"}:
|
|
134
|
+
return Column("numerical", representation="int", decimals=0)
|
|
135
|
+
if t in {"decimal", "numeric", "number", "money", "smallmoney"}:
|
|
136
|
+
scale = 2 if t in {"money", "smallmoney"} else 0
|
|
137
|
+
if args:
|
|
138
|
+
bits = [b.strip() for b in args.split(",")]
|
|
139
|
+
if len(bits) > 1 and bits[1].isdigit():
|
|
140
|
+
scale = int(bits[1])
|
|
141
|
+
if t == "number" and not args:
|
|
142
|
+
scale = 2
|
|
143
|
+
return Column("numerical", representation="int" if scale == 0 else "float", decimals=scale)
|
|
144
|
+
if t in {"float", "real", "double", "double precision", "float4", "float8"}:
|
|
145
|
+
return Column("numerical", representation="float", decimals=2)
|
|
146
|
+
if t in {"uuid", "uniqueidentifier"}:
|
|
147
|
+
return Column("id")
|
|
148
|
+
# character types
|
|
149
|
+
for pattern, sdtype in _PII_NAME_RULES:
|
|
150
|
+
if pattern.search(lname):
|
|
151
|
+
return Column(sdtype)
|
|
152
|
+
if lname in {"description", "comment", "comments", "notes", "note", "remarks", "review", "body", "message", "text"}:
|
|
153
|
+
return Column("text")
|
|
154
|
+
if t in {"text", "clob", "ntext", "longtext", "mediumtext"}:
|
|
155
|
+
return Column("text")
|
|
156
|
+
if lname.endswith(("_id", "_code", "_key", "_uuid")) or lname in {"code", "sku"}:
|
|
157
|
+
return Column("id")
|
|
158
|
+
if lname in _GEO_VALUES:
|
|
159
|
+
return Column("categorical", values=list(_GEO_VALUES[lname]))
|
|
160
|
+
if lname in {"company", "company_name", "employer"}:
|
|
161
|
+
return Column("company")
|
|
162
|
+
return Column("categorical")
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def parse_ddl(ddl: str | Path) -> Metadata:
|
|
166
|
+
"""Parse ``CREATE TABLE`` statements into :class:`Metadata` (with column hints and rules).
|
|
167
|
+
|
|
168
|
+
Understands column types, ``PRIMARY KEY``, ``REFERENCES`` / ``FOREIGN KEY``, ``NOT NULL``,
|
|
169
|
+
``ENUM(...)``, and ``CHECK`` rules (``IN``, ``BETWEEN``, comparisons with numbers, and
|
|
170
|
+
column-to-column comparisons).
|
|
171
|
+
"""
|
|
172
|
+
if isinstance(ddl, Path) or (isinstance(ddl, str) and ddl.strip().lower().endswith(".sql") and "\n" not in ddl):
|
|
173
|
+
ddl = Path(ddl).read_text()
|
|
174
|
+
sql = _strip_comments(str(ddl))
|
|
175
|
+
md = Metadata()
|
|
176
|
+
pending_fks: list[tuple[str, str, str, str | None]] = []
|
|
177
|
+
for m in _CREATE_RE.finditer(sql):
|
|
178
|
+
table_name = _ident(m.group(1))
|
|
179
|
+
body, _ = _balanced(sql, m.end() - 1)
|
|
180
|
+
table = md.add_table(table_name)
|
|
181
|
+
nullable: dict[str, bool] = {}
|
|
182
|
+
for item in _split_top(body):
|
|
183
|
+
if not item:
|
|
184
|
+
continue
|
|
185
|
+
upper = item.upper()
|
|
186
|
+
item_wo_constraint = re.sub(r"^CONSTRAINT\s+[\w\"`\[\]]+\s+", "", item, flags=re.IGNORECASE)
|
|
187
|
+
upper_wo = item_wo_constraint.upper()
|
|
188
|
+
if upper_wo.startswith("PRIMARY KEY"):
|
|
189
|
+
cols = re.search(r"\(([^)]*)\)", item_wo_constraint)
|
|
190
|
+
keys = [_ident(c) for c in cols.group(1).split(",")] if cols else []
|
|
191
|
+
if len(keys) == 1:
|
|
192
|
+
table.primary_key = keys[0]
|
|
193
|
+
continue
|
|
194
|
+
if upper_wo.startswith("FOREIGN KEY"):
|
|
195
|
+
cols = re.search(r"\(([^)]*)\)", item_wo_constraint)
|
|
196
|
+
ref = _REF_RE.search(item_wo_constraint)
|
|
197
|
+
if cols and ref:
|
|
198
|
+
fk_cols = [_ident(c) for c in cols.group(1).split(",")]
|
|
199
|
+
if len(fk_cols) == 1:
|
|
200
|
+
pending_fks.append((table_name, fk_cols[0], _ident(ref.group(1)),
|
|
201
|
+
_ident(ref.group(2)) if ref.group(2) else None))
|
|
202
|
+
continue
|
|
203
|
+
if upper_wo.startswith("CHECK"):
|
|
204
|
+
for expr in _checks(item_wo_constraint):
|
|
205
|
+
table.constraints.append(expr)
|
|
206
|
+
continue
|
|
207
|
+
if upper_wo.startswith(("UNIQUE", "INDEX", "KEY ", "FULLTEXT", "SPATIAL", "EXCLUDE")):
|
|
208
|
+
continue
|
|
209
|
+
# column definition
|
|
210
|
+
parts = item.split(None, 1)
|
|
211
|
+
col_name = _ident(parts[0])
|
|
212
|
+
rest = parts[1] if len(parts) > 1 else "varchar"
|
|
213
|
+
tm = _TYPE_RE.match(rest.strip())
|
|
214
|
+
sql_type, args = (tm.group(1), tm.group(2)) if tm else ("varchar", None)
|
|
215
|
+
column = _column_from_type(col_name, sql_type, args)
|
|
216
|
+
if "PRIMARY KEY" in upper:
|
|
217
|
+
table.primary_key = col_name
|
|
218
|
+
ref = _REF_RE.search(rest)
|
|
219
|
+
if ref:
|
|
220
|
+
pending_fks.append((table_name, col_name, _ident(ref.group(1)),
|
|
221
|
+
_ident(ref.group(2)) if ref.group(2) else None))
|
|
222
|
+
if re.search(r"\bUNIQUE\b", upper):
|
|
223
|
+
column.unique = True
|
|
224
|
+
nullable[col_name] = "NOT NULL" not in upper and "PRIMARY KEY" not in upper
|
|
225
|
+
table.columns[col_name] = column
|
|
226
|
+
for expr in _checks(rest):
|
|
227
|
+
table.constraints.append(expr)
|
|
228
|
+
# keys are ids
|
|
229
|
+
if table.primary_key and table.primary_key in table.columns:
|
|
230
|
+
_make_id(table.columns, table.primary_key)
|
|
231
|
+
for child, fk, parent, parent_col in pending_fks:
|
|
232
|
+
if parent not in md.tables:
|
|
233
|
+
raise ValueError(f"{child}.{fk} references unknown table {parent!r}")
|
|
234
|
+
_make_id(md.tables[child].columns, fk)
|
|
235
|
+
md.add_relationship(parent, child, fk, parent_col)
|
|
236
|
+
for table in md.tables.values():
|
|
237
|
+
_apply_rule_hints(table)
|
|
238
|
+
md.validate()
|
|
239
|
+
return md
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _make_id(columns: dict[str, Column], name: str) -> None:
|
|
243
|
+
old = columns[name]
|
|
244
|
+
rep = "int" if old.representation == "int" or old.sdtype == "numerical" else None
|
|
245
|
+
columns[name] = Column("id", representation=rep, regex_format=old.regex_format)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _apply_rule_hints(table: Table) -> None:
|
|
249
|
+
"""Fold single-column CHECK rules into column hints so generation respects them."""
|
|
250
|
+
for spec in table.constraints:
|
|
251
|
+
try:
|
|
252
|
+
constraints = parse_rule(spec) if isinstance(spec, str) else build_constraints([spec])
|
|
253
|
+
except ValueError:
|
|
254
|
+
continue
|
|
255
|
+
for c in constraints:
|
|
256
|
+
if isinstance(c, OneOf) and c.column in table.columns:
|
|
257
|
+
col = table.columns[c.column]
|
|
258
|
+
if col.sdtype in {"categorical", "id", "text"}:
|
|
259
|
+
table.columns[c.column] = Column("categorical", values=c.values)
|
|
260
|
+
else:
|
|
261
|
+
col.values = c.values
|
|
262
|
+
elif isinstance(c, Range) and c.column in table.columns:
|
|
263
|
+
col = table.columns[c.column]
|
|
264
|
+
if c.min is not None:
|
|
265
|
+
col.min_value = c.min if col.min_value is None else max(col.min_value, c.min)
|
|
266
|
+
if c.max is not None:
|
|
267
|
+
col.max_value = c.max if col.max_value is None else min(col.max_value, c.max)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
# ---------------------------------------------------------------------- generation
|
|
271
|
+
|
|
272
|
+
_GEO_VALUES = {
|
|
273
|
+
"country": ["United States", "United Kingdom", "Canada", "Germany", "India", "Australia", "France", "Brazil"],
|
|
274
|
+
"region": ["North", "South", "East", "West"],
|
|
275
|
+
"state": ["CA", "TX", "NY", "FL", "IL", "WA", "GA", "OH", "NC", "MA"],
|
|
276
|
+
"city": ["Springfield", "Riverton", "Fairview", "Franklin", "Greenville", "Madison", "Salem", "Arlington"],
|
|
277
|
+
"currency": ["USD", "EUR", "GBP", "INR", "CAD", "AUD"],
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
_NUMERIC_HINTS: list[tuple[re.Pattern, tuple[float, float]]] = [
|
|
281
|
+
(re.compile(r"(^|_)age($|_)"), (18, 90)),
|
|
282
|
+
(re.compile(r"(qty|quantity|units)"), (1, 20)),
|
|
283
|
+
(re.compile(r"(rating|stars|score)"), (1, 5)),
|
|
284
|
+
(re.compile(r"(percent|pct|rate|ratio|discount)"), (0, 0.5)),
|
|
285
|
+
(re.compile(r"(price|amount|cost|total|revenue|balance|salary|income|fee|value)"), (1, 1000)),
|
|
286
|
+
(re.compile(r"(^|_)year($|_)"), (2000, 2025)),
|
|
287
|
+
]
|
|
288
|
+
|
|
289
|
+
_DEFAULT_START = pd.Timestamp("2020-01-01")
|
|
290
|
+
_DEFAULT_END = pd.Timestamp("2025-12-31")
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _numeric_bounds(name: str, col: Column) -> tuple[float, float]:
|
|
294
|
+
lo, hi = col.min_value, col.max_value
|
|
295
|
+
if lo is None or hi is None:
|
|
296
|
+
for pattern, (dlo, dhi) in _NUMERIC_HINTS:
|
|
297
|
+
if pattern.search(name.lower()):
|
|
298
|
+
lo = dlo if lo is None else lo
|
|
299
|
+
hi = dhi if hi is None else hi
|
|
300
|
+
break
|
|
301
|
+
lo = 0 if lo is None else lo
|
|
302
|
+
hi = (lo + 1000 if lo >= 0 else 1000) if hi is None else hi
|
|
303
|
+
if hi < lo:
|
|
304
|
+
lo, hi = hi, lo
|
|
305
|
+
return float(lo), float(hi)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _with_nulls(values: np.ndarray, col: Column, rng: np.random.Generator) -> np.ndarray:
|
|
309
|
+
if col.null_rate:
|
|
310
|
+
values = values.astype(object)
|
|
311
|
+
values[rng.random(len(values)) < col.null_rate] = None
|
|
312
|
+
return values
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _weights(col: Column, k: int) -> np.ndarray | None:
|
|
316
|
+
if col.weights and len(col.weights) == k:
|
|
317
|
+
w = np.asarray(col.weights, dtype=float)
|
|
318
|
+
return w / w.sum()
|
|
319
|
+
return None
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def generate_column(name: str, col: Column, n: int, rng: np.random.Generator, locale=None,
|
|
323
|
+
table_name: str = "t") -> np.ndarray | pd.Series:
|
|
324
|
+
"""Generate ``n`` values for one column from its hints alone."""
|
|
325
|
+
if col.values and col.sdtype in {"categorical", "numerical", "boolean"}:
|
|
326
|
+
vals = np.array(col.values, dtype=object)
|
|
327
|
+
return _with_nulls(vals[rng.choice(len(vals), size=n, p=_weights(col, len(vals)))], col, rng)
|
|
328
|
+
if col.sdtype == "numerical":
|
|
329
|
+
lo, hi = _numeric_bounds(name, col)
|
|
330
|
+
x = lo + (hi - lo) * rng.beta(2.0, 2.0, size=n)
|
|
331
|
+
if col.representation == "int" or col.decimals == 0:
|
|
332
|
+
x = np.clip(np.round(x), np.ceil(lo), np.floor(hi)).astype(np.int64)
|
|
333
|
+
return _with_nulls(x, col, rng) if col.null_rate else x
|
|
334
|
+
x = np.round(x, col.decimals if col.decimals is not None else 2)
|
|
335
|
+
return _with_nulls(x, col, rng)
|
|
336
|
+
if col.sdtype == "boolean":
|
|
337
|
+
p = _weights(col, 2)
|
|
338
|
+
x = rng.random(n) < (p[0] if p is not None else 0.5)
|
|
339
|
+
return _with_nulls(x, col, rng) if col.null_rate else x
|
|
340
|
+
if col.sdtype == "datetime":
|
|
341
|
+
start = pd.Timestamp(col.min_value) if col.min_value is not None else _DEFAULT_START
|
|
342
|
+
end = pd.Timestamp(col.max_value) if col.max_value is not None else _DEFAULT_END
|
|
343
|
+
secs = rng.uniform(start.value / 1e9, end.value / 1e9, size=n)
|
|
344
|
+
dt = pd.to_datetime(pd.Series(secs), unit="s")
|
|
345
|
+
dt = dt.dt.floor("D") if col.representation == "date" else dt.dt.round("s")
|
|
346
|
+
if col.datetime_format:
|
|
347
|
+
dt = dt.dt.strftime(col.datetime_format)
|
|
348
|
+
out = dt.to_numpy()
|
|
349
|
+
return _with_nulls(out, col, rng) if col.null_rate else out
|
|
350
|
+
if col.sdtype == "categorical":
|
|
351
|
+
k = 5
|
|
352
|
+
vals = np.array([f"{name}_{i + 1}" for i in range(k)], dtype=object)
|
|
353
|
+
return _with_nulls(vals[rng.integers(0, k, size=n)], col, rng)
|
|
354
|
+
if col.sdtype == "text":
|
|
355
|
+
gen = TextGenerator(locale)
|
|
356
|
+
return _with_nulls(gen.generate(n, rng), col, rng)
|
|
357
|
+
if col.sdtype == "id":
|
|
358
|
+
return generate_keys(name, col, n, rng, table_name)
|
|
359
|
+
return _with_nulls(generate_pii(col.sdtype, n, rng, locale), col, rng)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def generate_keys(name: str, col: Column, n: int, rng: np.random.Generator, table_name: str = "t") -> np.ndarray:
|
|
363
|
+
if col.representation == "int":
|
|
364
|
+
return np.arange(1, n + 1, dtype=np.int64)
|
|
365
|
+
if col.regex_format:
|
|
366
|
+
return generate_from_regex(col.regex_format, n, rng, unique=True)
|
|
367
|
+
prefix = re.sub(r"[^A-Za-z]", "", table_name)[:3].upper() or "ID"
|
|
368
|
+
width = max(6, len(str(n)))
|
|
369
|
+
return np.array([f"{prefix}-{i:0{width}d}" for i in range(1, n + 1)], dtype=object)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
class SchemaSynthesizer:
|
|
373
|
+
"""Generates data from metadata alone: no seed data, no learning step.
|
|
374
|
+
|
|
375
|
+
Column hints (``values``, ``weights``, ``min_value``, ``max_value``, ``decimals``,
|
|
376
|
+
``null_rate``, ``regex_format``, ``datetime_format``) shape each column. Foreign keys
|
|
377
|
+
always point at generated parent rows, and every rule in the metadata holds on every row.
|
|
378
|
+
"""
|
|
379
|
+
|
|
380
|
+
def __init__(
|
|
381
|
+
self,
|
|
382
|
+
metadata: Metadata,
|
|
383
|
+
*,
|
|
384
|
+
constraints: Mapping[str, list[Constraint | Mapping | str]] | None = None,
|
|
385
|
+
children_per_parent: float | Mapping[str, float] = 3.0,
|
|
386
|
+
locale: str | list[str] | None = None,
|
|
387
|
+
) -> None:
|
|
388
|
+
metadata.validate()
|
|
389
|
+
self.metadata = metadata
|
|
390
|
+
self.extra_constraints = dict(constraints or {})
|
|
391
|
+
self.children_per_parent = children_per_parent
|
|
392
|
+
self.locale = locale
|
|
393
|
+
|
|
394
|
+
@classmethod
|
|
395
|
+
def from_ddl(cls, ddl: str | Path, **kwargs: Any) -> SchemaSynthesizer:
|
|
396
|
+
return cls(parse_ddl(ddl), **kwargs)
|
|
397
|
+
|
|
398
|
+
def _children(self, table: str) -> float:
|
|
399
|
+
if isinstance(self.children_per_parent, Mapping):
|
|
400
|
+
return float(self.children_per_parent.get(table, 3.0))
|
|
401
|
+
return float(self.children_per_parent)
|
|
402
|
+
|
|
403
|
+
def sample(self, num_rows: int | Mapping[str, int] = 100, seed: int | None = None) -> dict[str, pd.DataFrame]:
|
|
404
|
+
"""Generate every table. ``num_rows`` is the size of root tables (or a dict per table);
|
|
405
|
+
child tables default to ``children_per_parent`` rows per parent row."""
|
|
406
|
+
rng = np.random.default_rng(seed)
|
|
407
|
+
md = self.metadata
|
|
408
|
+
sizes = dict(num_rows) if isinstance(num_rows, Mapping) else {}
|
|
409
|
+
default_root = num_rows if isinstance(num_rows, int) else 100
|
|
410
|
+
out: dict[str, pd.DataFrame] = {}
|
|
411
|
+
for name in md.topological_order():
|
|
412
|
+
table = md.tables[name]
|
|
413
|
+
parents = md.parent_relationships(name)
|
|
414
|
+
if name in sizes:
|
|
415
|
+
n = int(sizes[name])
|
|
416
|
+
elif parents:
|
|
417
|
+
n = int(round(len(out[parents[0].parent_table]) * self._children(name)))
|
|
418
|
+
else:
|
|
419
|
+
n = int(default_root)
|
|
420
|
+
fks = {r.child_foreign_key: r for r in md.relationships if r.child_table == name}
|
|
421
|
+
data: dict[str, Any] = {}
|
|
422
|
+
for col_name, col in table.columns.items():
|
|
423
|
+
if col_name in fks:
|
|
424
|
+
continue
|
|
425
|
+
if col_name == table.primary_key:
|
|
426
|
+
data[col_name] = generate_keys(col_name, col, n, rng, name)
|
|
427
|
+
else:
|
|
428
|
+
data[col_name] = generate_column(col_name, col, n, rng, self.locale, name)
|
|
429
|
+
df = pd.DataFrame(data, index=pd.RangeIndex(n))
|
|
430
|
+
for fk, rel in fks.items():
|
|
431
|
+
if rel.parent_table == name:
|
|
432
|
+
keys = df[table.primary_key].to_numpy()
|
|
433
|
+
else:
|
|
434
|
+
keys = out[rel.parent_table][rel.parent_primary_key].to_numpy()
|
|
435
|
+
if len(keys) == 0:
|
|
436
|
+
df[fk] = None
|
|
437
|
+
continue
|
|
438
|
+
picked = keys[rng.integers(0, len(keys), size=n)]
|
|
439
|
+
col = table.columns[fk]
|
|
440
|
+
df[fk] = _with_nulls(picked, col, rng) if col.null_rate else picked
|
|
441
|
+
df = df[list(table.columns)]
|
|
442
|
+
df = self._enforce(name, table, df, rng)
|
|
443
|
+
out[name] = df.reset_index(drop=True)
|
|
444
|
+
return out
|
|
445
|
+
|
|
446
|
+
def _enforce(self, name: str, table: Table, df: pd.DataFrame, rng: np.random.Generator) -> pd.DataFrame:
|
|
447
|
+
constraints = build_constraints(table.constraints) + build_constraints(self.extra_constraints.get(name))
|
|
448
|
+
for c in constraints:
|
|
449
|
+
if any(col not in df.columns for col in c.columns):
|
|
450
|
+
continue
|
|
451
|
+
c.fit(df, table)
|
|
452
|
+
if isinstance(c, Inequality):
|
|
453
|
+
df = _repair_inequality(c, df, table, rng)
|
|
454
|
+
else:
|
|
455
|
+
df = c.repair(df)
|
|
456
|
+
if not c.is_valid(df).all():
|
|
457
|
+
df = df[c.is_valid(df)]
|
|
458
|
+
return df
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _repair_inequality(c: Inequality, df: pd.DataFrame, table: Table, rng: np.random.Generator) -> pd.DataFrame:
|
|
462
|
+
"""Rebuild ``high`` as ``low`` plus a positive gap wherever the rule fails."""
|
|
463
|
+
bad = ~c.is_valid(df)
|
|
464
|
+
if not bad.any():
|
|
465
|
+
return df
|
|
466
|
+
df = df.copy()
|
|
467
|
+
low_col, high_col = table.columns.get(c.low), table.columns.get(c.high)
|
|
468
|
+
if low_col is not None and low_col.sdtype == "datetime":
|
|
469
|
+
low = pd.to_datetime(df.loc[bad, c.low])
|
|
470
|
+
days = rng.integers(0 if not c.strict else 1, 15, size=int(bad.sum()))
|
|
471
|
+
high = low + pd.to_timedelta(days, unit="D")
|
|
472
|
+
if high_col is not None and high_col.representation == "date":
|
|
473
|
+
high = high.dt.floor("D")
|
|
474
|
+
fmt = high_col.datetime_format if high_col is not None else None
|
|
475
|
+
df.loc[bad, c.high] = high.dt.strftime(fmt).to_numpy() if fmt else high.to_numpy()
|
|
476
|
+
else:
|
|
477
|
+
low = pd.to_numeric(df.loc[bad, c.low], errors="coerce").to_numpy(dtype=float)
|
|
478
|
+
span = max(float(np.nanstd(pd.to_numeric(df[c.low], errors="coerce"))), 1.0)
|
|
479
|
+
gap = rng.uniform(0 if not c.strict else 1e-6, span, size=len(low))
|
|
480
|
+
high = low + gap
|
|
481
|
+
if high_col is not None and (high_col.representation == "int" or high_col.decimals == 0):
|
|
482
|
+
high = np.ceil(high)
|
|
483
|
+
elif high_col is not None and high_col.decimals is not None:
|
|
484
|
+
high = np.round(high, high_col.decimals)
|
|
485
|
+
df[c.high] = df[c.high].astype(float) if not pd.api.types.is_float_dtype(df[c.high]) else df[c.high]
|
|
486
|
+
df.loc[bad, c.high] = high
|
|
487
|
+
if high_col is not None and (high_col.representation == "int" or high_col.decimals == 0):
|
|
488
|
+
if df[c.high].notna().all():
|
|
489
|
+
df[c.high] = df[c.high].astype(np.int64)
|
|
490
|
+
return df
|