synthschema 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synthschema/__init__.py +56 -0
- synthschema/_version.py +1 -0
- synthschema/cli.py +134 -0
- synthschema/constraints.py +438 -0
- synthschema/copula.py +79 -0
- synthschema/datasets.py +106 -0
- synthschema/evaluation.py +446 -0
- synthschema/generators.py +456 -0
- synthschema/marginals.py +143 -0
- synthschema/metadata.py +547 -0
- synthschema/multi_table.py +272 -0
- synthschema/py.typed +0 -0
- synthschema/schema.py +490 -0
- synthschema/single_table.py +311 -0
- synthschema/transformers.py +247 -0
- synthschema-0.1.0.dist-info/METADATA +253 -0
- synthschema-0.1.0.dist-info/RECORD +20 -0
- synthschema-0.1.0.dist-info/WHEEL +4 -0
- synthschema-0.1.0.dist-info/entry_points.txt +2 -0
- synthschema-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
"""Value generators for columns that are never learned: keys, PII and free text.
|
|
2
|
+
|
|
3
|
+
Real values of these columns never reach a model and are never copied to the output.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
from collections.abc import Iterable
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
import pandas as pd
|
|
13
|
+
|
|
14
|
+
from .metadata import Column
|
|
15
|
+
|
|
16
|
+
# ---------------------------------------------------------------------- regex patterns
|
|
17
|
+
|
|
18
|
+
_CLASS_CHARS = {
|
|
19
|
+
"0-9": "0123456789",
|
|
20
|
+
"A-Z": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
|
|
21
|
+
"a-z": "abcdefghijklmnopqrstuvwxyz",
|
|
22
|
+
}
|
|
23
|
+
_TOKEN_RE = re.compile(r"\[([^\]]+)\]|\\d|\\w|\\(.)|(.)")
|
|
24
|
+
_QUANT_RE = re.compile(r"\{(\d+)(?:,(\d+))?\}")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _expand_class(spec: str) -> str:
|
|
28
|
+
chars: list[str] = []
|
|
29
|
+
i = 0
|
|
30
|
+
while i < len(spec):
|
|
31
|
+
if i + 2 < len(spec) and spec[i + 1] == "-":
|
|
32
|
+
start, end = ord(spec[i]), ord(spec[i + 2])
|
|
33
|
+
chars.extend(chr(c) for c in range(start, end + 1))
|
|
34
|
+
i += 3
|
|
35
|
+
else:
|
|
36
|
+
chars.append(spec[i])
|
|
37
|
+
i += 1
|
|
38
|
+
return "".join(dict.fromkeys(chars))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def parse_regex(pattern: str) -> list[tuple[str, int, int]]:
|
|
42
|
+
"""Parse the small regex subset synthschema generates from.
|
|
43
|
+
|
|
44
|
+
Supported: literals, escaped literals, ``\\d``, ``\\w``, character classes such as
|
|
45
|
+
``[A-Z]`` or ``[A-Za-z0-9_]``, and the quantifiers ``{n}`` and ``{m,n}``.
|
|
46
|
+
"""
|
|
47
|
+
tokens: list[tuple[str, int, int]] = []
|
|
48
|
+
pos = 0
|
|
49
|
+
pattern = pattern.strip("^$")
|
|
50
|
+
while pos < len(pattern):
|
|
51
|
+
m = _TOKEN_RE.match(pattern, pos)
|
|
52
|
+
if m is None: # pragma: no cover - the token regex matches any character
|
|
53
|
+
raise ValueError(f"Cannot parse regex {pattern!r} at {pos}")
|
|
54
|
+
text = m.group(0)
|
|
55
|
+
if m.group(1) is not None:
|
|
56
|
+
chars = _expand_class(m.group(1))
|
|
57
|
+
elif text == r"\d":
|
|
58
|
+
chars = _CLASS_CHARS["0-9"]
|
|
59
|
+
elif text == r"\w":
|
|
60
|
+
chars = _CLASS_CHARS["A-Z"] + _CLASS_CHARS["a-z"] + _CLASS_CHARS["0-9"] + "_"
|
|
61
|
+
elif m.group(2) is not None:
|
|
62
|
+
chars = m.group(2)
|
|
63
|
+
else:
|
|
64
|
+
chars = m.group(3)
|
|
65
|
+
pos = m.end()
|
|
66
|
+
lo = hi = 1
|
|
67
|
+
q = _QUANT_RE.match(pattern, pos)
|
|
68
|
+
if q:
|
|
69
|
+
lo = int(q.group(1))
|
|
70
|
+
hi = int(q.group(2)) if q.group(2) else lo
|
|
71
|
+
pos = q.end()
|
|
72
|
+
tokens.append((chars, lo, hi))
|
|
73
|
+
return tokens
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _capacity(tokens: list[tuple[str, int, int]]) -> float:
|
|
77
|
+
cap = 1.0
|
|
78
|
+
for chars, _lo, hi in tokens:
|
|
79
|
+
cap *= float(len(chars)) ** hi
|
|
80
|
+
if cap > 1e18:
|
|
81
|
+
return cap
|
|
82
|
+
return cap
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _widen(tokens: list[tuple[str, int, int]], needed: int) -> list[tuple[str, int, int]]:
|
|
86
|
+
"""Grow the last variable token until the pattern can hold ``needed`` distinct values."""
|
|
87
|
+
tokens = list(tokens)
|
|
88
|
+
variable = [i for i, (c, _, _) in enumerate(tokens) if len(c) > 1]
|
|
89
|
+
if not variable:
|
|
90
|
+
tokens.append((_CLASS_CHARS["0-9"], 1, 1))
|
|
91
|
+
variable = [len(tokens) - 1]
|
|
92
|
+
i = variable[-1]
|
|
93
|
+
while _capacity(tokens) < needed:
|
|
94
|
+
chars, lo, hi = tokens[i]
|
|
95
|
+
tokens[i] = (chars, lo + 1, hi + 1)
|
|
96
|
+
return tokens
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _generate_tokens(tokens: list[tuple[str, int, int]], n: int, rng: np.random.Generator) -> np.ndarray:
|
|
100
|
+
if n == 0:
|
|
101
|
+
return np.array([], dtype=object)
|
|
102
|
+
if all(lo == hi for _, lo, hi in tokens):
|
|
103
|
+
parts = []
|
|
104
|
+
for chars, k, _ in tokens:
|
|
105
|
+
if k == 0:
|
|
106
|
+
continue
|
|
107
|
+
pool = np.array(list(chars))
|
|
108
|
+
parts.append(pool[rng.integers(0, len(pool), size=(n, k))])
|
|
109
|
+
if not parts:
|
|
110
|
+
return np.full(n, "", dtype=object)
|
|
111
|
+
mat = np.ascontiguousarray(np.concatenate(parts, axis=1))
|
|
112
|
+
width = mat.shape[1]
|
|
113
|
+
return mat.view(f"<U{width}").ravel().astype(object)
|
|
114
|
+
out = []
|
|
115
|
+
for _ in range(n):
|
|
116
|
+
pieces = []
|
|
117
|
+
for chars, lo, hi in tokens:
|
|
118
|
+
k = int(rng.integers(lo, hi + 1))
|
|
119
|
+
pieces.append("".join(chars[j] for j in rng.integers(0, len(chars), size=k)))
|
|
120
|
+
out.append("".join(pieces))
|
|
121
|
+
return np.array(out, dtype=object)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def generate_from_regex(
|
|
125
|
+
pattern: str, n: int, rng: np.random.Generator, unique: bool = True
|
|
126
|
+
) -> np.ndarray:
|
|
127
|
+
"""Generate ``n`` strings that match ``pattern`` (unique by default)."""
|
|
128
|
+
tokens = parse_regex(pattern)
|
|
129
|
+
if not unique:
|
|
130
|
+
return _generate_tokens(tokens, n, rng)
|
|
131
|
+
if _capacity(tokens) < 2 * n:
|
|
132
|
+
tokens = _widen(tokens, 2 * n)
|
|
133
|
+
values = pd.unique(_generate_tokens(tokens, n, rng))
|
|
134
|
+
for _ in range(50):
|
|
135
|
+
if len(values) >= n:
|
|
136
|
+
break
|
|
137
|
+
extra = _generate_tokens(tokens, 2 * (n - len(values)) + 10, rng)
|
|
138
|
+
values = pd.unique(np.concatenate([values, extra]))
|
|
139
|
+
if len(values) < n: # pragma: no cover - only with tiny pattern spaces
|
|
140
|
+
raise ValueError(f"Could not generate {n} unique values for pattern {pattern!r}")
|
|
141
|
+
return np.asarray(values[:n], dtype=object)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def infer_regex(values: Iterable, max_samples: int = 2000) -> str | None:
|
|
145
|
+
"""Learn a regex that describes a set of identifier strings, e.g. ``SKU-[0-9]{5}``."""
|
|
146
|
+
vals = [str(v) for v in values]
|
|
147
|
+
if not vals:
|
|
148
|
+
return None
|
|
149
|
+
if len(vals) > max_samples:
|
|
150
|
+
idx = np.random.default_rng(0).choice(len(vals), max_samples, replace=False)
|
|
151
|
+
vals = [vals[i] for i in idx]
|
|
152
|
+
lengths = {len(v) for v in vals}
|
|
153
|
+
if len(lengths) == 1:
|
|
154
|
+
length = lengths.pop()
|
|
155
|
+
position_chars = [{v[i] for v in vals} for i in range(length)]
|
|
156
|
+
# A run of digit positions is a number: if any digit in the run varies, the whole
|
|
157
|
+
# run is variable (so zero-padded counters such as 00042 become [0-9]{5}).
|
|
158
|
+
numeric_run = [False] * length
|
|
159
|
+
i = 0
|
|
160
|
+
while i < length:
|
|
161
|
+
if all(c.isdigit() for c in position_chars[i]):
|
|
162
|
+
j = i
|
|
163
|
+
while j < length and all(c.isdigit() for c in position_chars[j]):
|
|
164
|
+
j += 1
|
|
165
|
+
if any(len(position_chars[k]) > 1 for k in range(i, j)):
|
|
166
|
+
for k in range(i, j):
|
|
167
|
+
numeric_run[k] = True
|
|
168
|
+
i = j
|
|
169
|
+
else:
|
|
170
|
+
i += 1
|
|
171
|
+
tokens: list[str] = []
|
|
172
|
+
for i in range(length):
|
|
173
|
+
chars = position_chars[i]
|
|
174
|
+
if numeric_run[i]:
|
|
175
|
+
tokens.append("[0-9]")
|
|
176
|
+
elif len(chars) == 1:
|
|
177
|
+
tokens.append(re.escape(next(iter(chars))))
|
|
178
|
+
elif all(c.isdigit() for c in chars):
|
|
179
|
+
tokens.append("[0-9]")
|
|
180
|
+
elif all(c.isupper() and c.isalpha() for c in chars):
|
|
181
|
+
tokens.append("[A-Z]")
|
|
182
|
+
elif all(c.islower() and c.isalpha() for c in chars):
|
|
183
|
+
tokens.append("[a-z]")
|
|
184
|
+
elif all(c.isalnum() for c in chars):
|
|
185
|
+
tokens.append("[A-Za-z0-9]")
|
|
186
|
+
else:
|
|
187
|
+
return None
|
|
188
|
+
return _compress(tokens)
|
|
189
|
+
prefix = _common_prefix(vals)
|
|
190
|
+
rests = [v[len(prefix):] for v in vals]
|
|
191
|
+
width = max(len(r) for r in rests)
|
|
192
|
+
if all(r.isdigit() for r in rests if r):
|
|
193
|
+
return re.escape(prefix) + f"[0-9]{{{width}}}"
|
|
194
|
+
if all(r.isalnum() for r in rests if r):
|
|
195
|
+
return re.escape(prefix) + f"[A-Za-z0-9]{{{width}}}"
|
|
196
|
+
return None
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _compress(tokens: list[str]) -> str:
|
|
200
|
+
out: list[str] = []
|
|
201
|
+
i = 0
|
|
202
|
+
while i < len(tokens):
|
|
203
|
+
j = i
|
|
204
|
+
while j + 1 < len(tokens) and tokens[j + 1] == tokens[i]:
|
|
205
|
+
j += 1
|
|
206
|
+
run = j - i + 1
|
|
207
|
+
tok = tokens[i]
|
|
208
|
+
if tok.startswith("[") and run > 1:
|
|
209
|
+
out.append(f"{tok}{{{run}}}")
|
|
210
|
+
else:
|
|
211
|
+
out.append(tok * run)
|
|
212
|
+
i = j + 1
|
|
213
|
+
return "".join(out)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _common_prefix(vals: list[str]) -> str:
|
|
217
|
+
prefix = vals[0]
|
|
218
|
+
for v in vals[1:]:
|
|
219
|
+
while not v.startswith(prefix):
|
|
220
|
+
prefix = prefix[:-1]
|
|
221
|
+
if not prefix:
|
|
222
|
+
return ""
|
|
223
|
+
# Do not swallow a varying digit run into the prefix.
|
|
224
|
+
while prefix and prefix[-1].isdigit():
|
|
225
|
+
prefix = prefix[:-1]
|
|
226
|
+
return prefix
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
# ---------------------------------------------------------------------- id generator
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
class IdGenerator:
|
|
233
|
+
"""Generates key values that look like the real ones but never copy them."""
|
|
234
|
+
|
|
235
|
+
def __init__(self, column: Column | None = None) -> None:
|
|
236
|
+
self.column = column or Column("id")
|
|
237
|
+
self.kind = "int"
|
|
238
|
+
self.start = 1
|
|
239
|
+
self.pattern: str | None = self.column.regex_format
|
|
240
|
+
self.unique_ratio = 1.0
|
|
241
|
+
self.null_rate = 0.0
|
|
242
|
+
|
|
243
|
+
def fit(self, series: pd.Series) -> IdGenerator:
|
|
244
|
+
nonnull = series.dropna()
|
|
245
|
+
self.null_rate = float(series.isna().mean()) if len(series) else 0.0
|
|
246
|
+
self.unique_ratio = float(nonnull.nunique() / len(nonnull)) if len(nonnull) else 1.0
|
|
247
|
+
if self.column.representation == "int" or (
|
|
248
|
+
pd.api.types.is_numeric_dtype(series) and not pd.api.types.is_bool_dtype(series)
|
|
249
|
+
):
|
|
250
|
+
self.kind = "int"
|
|
251
|
+
self.start = int(nonnull.min()) if len(nonnull) else 1
|
|
252
|
+
return self
|
|
253
|
+
if self.pattern is None and len(nonnull):
|
|
254
|
+
sample = nonnull.astype(str)
|
|
255
|
+
if sample.str.match(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$").all():
|
|
256
|
+
self.kind = "uuid"
|
|
257
|
+
return self
|
|
258
|
+
self.pattern = infer_regex(sample)
|
|
259
|
+
self.kind = "regex" if self.pattern else "uuid"
|
|
260
|
+
return self
|
|
261
|
+
|
|
262
|
+
def generate(
|
|
263
|
+
self, n: int, rng: np.random.Generator, unique: bool | None = None, allow_nulls: bool = True
|
|
264
|
+
) -> np.ndarray:
|
|
265
|
+
if unique is None:
|
|
266
|
+
unique = self.unique_ratio >= 0.999
|
|
267
|
+
k = n if unique else max(1, int(round(self.unique_ratio * n)))
|
|
268
|
+
if self.kind == "int":
|
|
269
|
+
pool = np.arange(self.start, self.start + k, dtype=np.int64)
|
|
270
|
+
elif self.kind == "uuid":
|
|
271
|
+
raw = rng.integers(0, 16, size=(k, 32))
|
|
272
|
+
hexchars = np.array(list("0123456789abcdef"))[raw]
|
|
273
|
+
strings = np.ascontiguousarray(hexchars).view("<U32").ravel()
|
|
274
|
+
pool = np.array(
|
|
275
|
+
[f"{s[:8]}-{s[8:12]}-4{s[13:16]}-{s[16:20]}-{s[20:]}" for s in strings], dtype=object
|
|
276
|
+
)
|
|
277
|
+
else:
|
|
278
|
+
pool = generate_from_regex(self.pattern, k, rng, unique=True)
|
|
279
|
+
values = pool if unique else pool[rng.integers(0, len(pool), size=n)]
|
|
280
|
+
if allow_nulls and self.null_rate > 0:
|
|
281
|
+
values = values.astype(object)
|
|
282
|
+
values[rng.random(n) < self.null_rate] = None
|
|
283
|
+
return values
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
# ---------------------------------------------------------------------- PII and text
|
|
287
|
+
|
|
288
|
+
_FIRST = (
|
|
289
|
+
"James Mary Robert Patricia John Jennifer Michael Linda David Elizabeth William Barbara "
|
|
290
|
+
"Richard Susan Joseph Jessica Thomas Sarah Charles Karen Daniel Lisa Matthew Nancy Anthony "
|
|
291
|
+
"Sandra Mark Ashley Priya Arjun Wei Mei Omar Fatima Lucas Sofia Mateo Aisha Kenji Yuki"
|
|
292
|
+
).split()
|
|
293
|
+
_LAST = (
|
|
294
|
+
"Smith Johnson Williams Brown Jones Garcia Miller Davis Rodriguez Martinez Hernandez Lopez "
|
|
295
|
+
"Gonzalez Wilson Anderson Thomas Taylor Moore Jackson Martin Lee Perez Thompson White Harris "
|
|
296
|
+
"Sanchez Clark Ramirez Lewis Robinson Patel Kumar Chen Wang Nguyen Kim Ali Khan Silva Rossi"
|
|
297
|
+
).split()
|
|
298
|
+
_STREETS = "Main Oak Pine Maple Cedar Elm Washington Lake Hill Park River Sunset Highland".split()
|
|
299
|
+
_SUFFIX = "St Ave Rd Blvd Ln Dr Way Ct".split()
|
|
300
|
+
_CITIES = (
|
|
301
|
+
"Springfield Riverton Fairview Georgetown Franklin Greenville Bristol Clinton Madison "
|
|
302
|
+
"Salem Arlington Ashland Dover Hudson Kingston"
|
|
303
|
+
).split()
|
|
304
|
+
_COMPANY_A = "Blue North Bright Prime Summit Silver Global Apex Pioneer Vertex Harbor Cedar".split()
|
|
305
|
+
_COMPANY_B = "Systems Labs Partners Holdings Analytics Logistics Foods Retail Health Energy".split()
|
|
306
|
+
_WORDS = (
|
|
307
|
+
"data quality order customer product service delivery price value report account team "
|
|
308
|
+
"support request update review payment store region market growth plan system process "
|
|
309
|
+
"issue result project network design contract offer return good fast late great poor "
|
|
310
|
+
"excellent average simple clear helpful friendly slow easy new standard premium"
|
|
311
|
+
).split()
|
|
312
|
+
|
|
313
|
+
_FAKER_METHODS = {
|
|
314
|
+
"email": "email",
|
|
315
|
+
"phone_number": "phone_number",
|
|
316
|
+
"name": "name",
|
|
317
|
+
"first_name": "first_name",
|
|
318
|
+
"last_name": "last_name",
|
|
319
|
+
"address": "address",
|
|
320
|
+
"street_address": "street_address",
|
|
321
|
+
"ssn": "ssn",
|
|
322
|
+
"credit_card_number": "credit_card_number",
|
|
323
|
+
"ip_address": "ipv4_private",
|
|
324
|
+
"url": "url",
|
|
325
|
+
"company": "company",
|
|
326
|
+
"postcode": "postcode",
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
_FAKER_MAX_DIRECT = 100_000
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _faker(locale: str | list[str] | None, rng: np.random.Generator):
|
|
333
|
+
try:
|
|
334
|
+
from faker import Faker
|
|
335
|
+
except ImportError:
|
|
336
|
+
return None
|
|
337
|
+
fake = Faker(locale)
|
|
338
|
+
fake.seed_instance(int(rng.integers(0, 2**31 - 1)))
|
|
339
|
+
return fake
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _pick(pool: list[str], n: int, rng: np.random.Generator) -> np.ndarray:
|
|
343
|
+
return np.array(pool, dtype=object)[rng.integers(0, len(pool), size=n)]
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _digits(n: int, k: int, rng: np.random.Generator) -> np.ndarray:
|
|
347
|
+
mat = np.array(list("0123456789"))[rng.integers(0, 10, size=(n, k))]
|
|
348
|
+
return np.ascontiguousarray(mat).view(f"<U{k}").ravel().astype(object)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _builtin_pii(sdtype: str, n: int, rng: np.random.Generator) -> np.ndarray:
|
|
352
|
+
first, last = _pick(_FIRST, n, rng), _pick(_LAST, n, rng)
|
|
353
|
+
if sdtype == "first_name":
|
|
354
|
+
return first
|
|
355
|
+
if sdtype == "last_name":
|
|
356
|
+
return last
|
|
357
|
+
if sdtype == "name":
|
|
358
|
+
return first + " " + last
|
|
359
|
+
if sdtype == "email":
|
|
360
|
+
num = _digits(n, 4, rng)
|
|
361
|
+
return np.char.lower((first + "." + last + num).astype(str)).astype(object) + "@example.com"
|
|
362
|
+
if sdtype == "phone_number":
|
|
363
|
+
return "+1-555-" + _digits(n, 3, rng) + "-" + _digits(n, 4, rng)
|
|
364
|
+
if sdtype in {"address", "street_address"}:
|
|
365
|
+
street = (
|
|
366
|
+
rng.integers(1, 9999, size=n).astype(str).astype(object)
|
|
367
|
+
+ " " + _pick(_STREETS, n, rng) + " " + _pick(_SUFFIX, n, rng)
|
|
368
|
+
)
|
|
369
|
+
if sdtype == "street_address":
|
|
370
|
+
return street
|
|
371
|
+
return street + ", " + _pick(_CITIES, n, rng) + " " + _digits(n, 5, rng)
|
|
372
|
+
if sdtype == "ssn": # 9xx area numbers are never issued
|
|
373
|
+
return "9" + _digits(n, 2, rng) + "-" + _digits(n, 2, rng) + "-" + _digits(n, 4, rng)
|
|
374
|
+
if sdtype == "credit_card_number": # 4000 0000 prefix is a well-known test range
|
|
375
|
+
return "40000000" + _digits(n, 8, rng)
|
|
376
|
+
if sdtype == "ip_address":
|
|
377
|
+
octets = rng.integers(0, 256, size=(n, 3)).astype(str)
|
|
378
|
+
return np.array(["10." + ".".join(r) for r in octets], dtype=object)
|
|
379
|
+
if sdtype == "url":
|
|
380
|
+
return "https://www." + np.char.lower(_pick(_COMPANY_A, n, rng).astype(str)).astype(object) + _digits(n, 3, rng) + ".example.com"
|
|
381
|
+
if sdtype == "company":
|
|
382
|
+
return _pick(_COMPANY_A, n, rng) + " " + _pick(_COMPANY_B, n, rng)
|
|
383
|
+
if sdtype == "postcode":
|
|
384
|
+
return _digits(n, 5, rng)
|
|
385
|
+
raise ValueError(f"No generator for sdtype {sdtype!r}")
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def generate_pii(
|
|
389
|
+
sdtype: str, n: int, rng: np.random.Generator, locale: str | list[str] | None = None
|
|
390
|
+
) -> np.ndarray:
|
|
391
|
+
"""Generate ``n`` fake values of a PII type. Uses Faker when installed."""
|
|
392
|
+
if n == 0:
|
|
393
|
+
return np.array([], dtype=object)
|
|
394
|
+
fake = _faker(locale, rng)
|
|
395
|
+
if fake is None:
|
|
396
|
+
return _builtin_pii(sdtype, n, rng)
|
|
397
|
+
method = getattr(fake, _FAKER_METHODS[sdtype])
|
|
398
|
+
size = min(n, _FAKER_MAX_DIRECT)
|
|
399
|
+
values = np.array([method() for _ in range(size)], dtype=object)
|
|
400
|
+
if sdtype == "address":
|
|
401
|
+
values = np.array([v.replace("\n", ", ") for v in values], dtype=object)
|
|
402
|
+
if size < n:
|
|
403
|
+
values = values[rng.integers(0, size, size=n)]
|
|
404
|
+
return values
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
class PiiGenerator:
|
|
408
|
+
def __init__(self, sdtype: str, locale: str | list[str] | None = None) -> None:
|
|
409
|
+
self.sdtype = sdtype
|
|
410
|
+
self.locale = locale
|
|
411
|
+
self.null_rate = 0.0
|
|
412
|
+
|
|
413
|
+
def fit(self, series: pd.Series) -> PiiGenerator:
|
|
414
|
+
self.null_rate = float(series.isna().mean()) if len(series) else 0.0
|
|
415
|
+
return self
|
|
416
|
+
|
|
417
|
+
def generate(self, n: int, rng: np.random.Generator) -> np.ndarray:
|
|
418
|
+
values = generate_pii(self.sdtype, n, rng, self.locale).astype(object)
|
|
419
|
+
if self.null_rate > 0:
|
|
420
|
+
values[rng.random(n) < self.null_rate] = None
|
|
421
|
+
return values
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
class TextGenerator:
|
|
425
|
+
"""Generates placeholder free text with a word-count profile like the real column."""
|
|
426
|
+
|
|
427
|
+
def __init__(self, locale: str | list[str] | None = None) -> None:
|
|
428
|
+
self.locale = locale
|
|
429
|
+
self.mean_words = 8.0
|
|
430
|
+
self.std_words = 3.0
|
|
431
|
+
self.null_rate = 0.0
|
|
432
|
+
|
|
433
|
+
def fit(self, series: pd.Series) -> TextGenerator:
|
|
434
|
+
self.null_rate = float(series.isna().mean()) if len(series) else 0.0
|
|
435
|
+
counts = series.dropna().astype(str).str.split().str.len()
|
|
436
|
+
if len(counts):
|
|
437
|
+
self.mean_words = float(max(counts.mean(), 1))
|
|
438
|
+
self.std_words = float(counts.std() if len(counts) > 1 else 0) or 1.0
|
|
439
|
+
return self
|
|
440
|
+
|
|
441
|
+
def generate(self, n: int, rng: np.random.Generator) -> np.ndarray:
|
|
442
|
+
lengths = np.clip(np.round(rng.normal(self.mean_words, self.std_words, n)), 1, 200).astype(int)
|
|
443
|
+
fake = _faker(self.locale, rng)
|
|
444
|
+
if fake is not None and n <= _FAKER_MAX_DIRECT:
|
|
445
|
+
values = np.array(
|
|
446
|
+
[fake.sentence(nb_words=int(k), variable_nb_words=False) for k in lengths], dtype=object
|
|
447
|
+
)
|
|
448
|
+
else:
|
|
449
|
+
words = np.array(_WORDS, dtype=object)
|
|
450
|
+
values = np.array(
|
|
451
|
+
[" ".join(words[rng.integers(0, len(words), size=k)]).capitalize() + "." for k in lengths],
|
|
452
|
+
dtype=object,
|
|
453
|
+
)
|
|
454
|
+
if self.null_rate > 0:
|
|
455
|
+
values[rng.random(n) < self.null_rate] = None
|
|
456
|
+
return values
|
synthschema/marginals.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""Univariate (marginal) distributions used by the Gaussian copula."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import warnings
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
from scipy import stats
|
|
9
|
+
|
|
10
|
+
EPS = 1e-6
|
|
11
|
+
_MAX_EMPIRICAL_POINTS = 50_000
|
|
12
|
+
|
|
13
|
+
DISTRIBUTIONS = ("empirical", "norm", "uniform", "beta", "truncnorm", "gamma", "auto")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Marginal:
|
|
17
|
+
name = "base"
|
|
18
|
+
|
|
19
|
+
def fit(self, x: np.ndarray) -> Marginal: # pragma: no cover - interface
|
|
20
|
+
raise NotImplementedError
|
|
21
|
+
|
|
22
|
+
def cdf(self, x: np.ndarray, rng: np.random.Generator | None = None) -> np.ndarray: # pragma: no cover
|
|
23
|
+
raise NotImplementedError
|
|
24
|
+
|
|
25
|
+
def ppf(self, u: np.ndarray) -> np.ndarray: # pragma: no cover
|
|
26
|
+
raise NotImplementedError
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Constant(Marginal):
|
|
30
|
+
name = "constant"
|
|
31
|
+
|
|
32
|
+
def fit(self, x):
|
|
33
|
+
self.value = float(x[0]) if len(x) else 0.0
|
|
34
|
+
return self
|
|
35
|
+
|
|
36
|
+
def cdf(self, x, rng=None):
|
|
37
|
+
return np.full(len(x), 0.5)
|
|
38
|
+
|
|
39
|
+
def ppf(self, u):
|
|
40
|
+
return np.full(len(u), self.value)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class Empirical(Marginal):
|
|
44
|
+
"""Exact empirical distribution with linear interpolation between observed quantiles.
|
|
45
|
+
|
|
46
|
+
Ties (discrete values) are spread uniformly over their probability mass so the
|
|
47
|
+
probability-integral transform stays uniform.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
name = "empirical"
|
|
51
|
+
|
|
52
|
+
def fit(self, x):
|
|
53
|
+
x = np.sort(np.asarray(x, dtype=float))
|
|
54
|
+
if len(x) > _MAX_EMPIRICAL_POINTS:
|
|
55
|
+
# x is sorted, so linear interpolation over ranks equals np.quantile (and is O(n)).
|
|
56
|
+
positions = np.linspace(0, len(x) - 1, _MAX_EMPIRICAL_POINTS)
|
|
57
|
+
x = np.interp(positions, np.arange(len(x)), x)
|
|
58
|
+
self.sorted = x
|
|
59
|
+
n = len(x)
|
|
60
|
+
self.probs = (np.arange(n) + 0.5) / n
|
|
61
|
+
return self
|
|
62
|
+
|
|
63
|
+
def cdf(self, x, rng=None):
|
|
64
|
+
x = np.asarray(x, dtype=float)
|
|
65
|
+
s, n = self.sorted, len(self.sorted)
|
|
66
|
+
left = np.searchsorted(s, x, side="left")
|
|
67
|
+
right = np.searchsorted(s, x, side="right")
|
|
68
|
+
tied = right > left
|
|
69
|
+
u = np.interp(x, s, self.probs)
|
|
70
|
+
if tied.any():
|
|
71
|
+
frac = rng.random(tied.sum()) if rng is not None else 0.5
|
|
72
|
+
u[tied] = (left[tied] + frac * (right[tied] - left[tied])) / n
|
|
73
|
+
return np.clip(u, EPS, 1 - EPS)
|
|
74
|
+
|
|
75
|
+
def ppf(self, u):
|
|
76
|
+
return np.interp(np.asarray(u, dtype=float), self.probs, self.sorted)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class Parametric(Marginal):
|
|
80
|
+
"""A scipy.stats distribution fitted by maximum likelihood."""
|
|
81
|
+
|
|
82
|
+
def __init__(self, name: str) -> None:
|
|
83
|
+
self.name = name
|
|
84
|
+
|
|
85
|
+
def fit(self, x):
|
|
86
|
+
x = np.asarray(x, dtype=float)
|
|
87
|
+
lo, hi = float(x.min()), float(x.max())
|
|
88
|
+
span = hi - lo
|
|
89
|
+
with warnings.catch_warnings():
|
|
90
|
+
warnings.simplefilter("ignore")
|
|
91
|
+
if self.name == "norm":
|
|
92
|
+
self.dist = stats.norm(*stats.norm.fit(x))
|
|
93
|
+
elif self.name == "uniform":
|
|
94
|
+
self.dist = stats.uniform(lo, span)
|
|
95
|
+
elif self.name == "beta":
|
|
96
|
+
loc, scale = lo - 1e-3 * span, span * 1.002
|
|
97
|
+
a, b, _, _ = stats.beta.fit(x, floc=loc, fscale=scale)
|
|
98
|
+
self.dist = stats.beta(a, b, loc=loc, scale=scale)
|
|
99
|
+
elif self.name == "truncnorm":
|
|
100
|
+
mu, sd = float(x.mean()), float(x.std()) or 1.0
|
|
101
|
+
self.dist = stats.truncnorm((lo - mu) / sd, (hi - mu) / sd, loc=mu, scale=sd)
|
|
102
|
+
elif self.name == "gamma":
|
|
103
|
+
a, loc, scale = stats.gamma.fit(x, floc=lo - 1e-3 * span)
|
|
104
|
+
self.dist = stats.gamma(a, loc=loc, scale=scale)
|
|
105
|
+
else:
|
|
106
|
+
raise ValueError(f"Unknown distribution {self.name!r}")
|
|
107
|
+
return self
|
|
108
|
+
|
|
109
|
+
def cdf(self, x, rng=None):
|
|
110
|
+
return np.clip(self.dist.cdf(np.asarray(x, dtype=float)), EPS, 1 - EPS)
|
|
111
|
+
|
|
112
|
+
def ppf(self, u):
|
|
113
|
+
return self.dist.ppf(np.asarray(u, dtype=float))
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def fit_marginal(x: np.ndarray, distribution: str = "empirical") -> Marginal:
|
|
117
|
+
"""Fit a marginal. ``"auto"`` picks the parametric family with the best KS statistic."""
|
|
118
|
+
x = np.asarray(x, dtype=float)
|
|
119
|
+
x = x[np.isfinite(x)]
|
|
120
|
+
if len(x) == 0 or np.all(x == x[0]):
|
|
121
|
+
return Constant().fit(x)
|
|
122
|
+
if distribution == "empirical":
|
|
123
|
+
return Empirical().fit(x)
|
|
124
|
+
if distribution == "auto":
|
|
125
|
+
sample = x if len(x) <= 5000 else np.random.default_rng(0).choice(x, 5000, replace=False)
|
|
126
|
+
best, best_stat = None, np.inf
|
|
127
|
+
for name in ("norm", "uniform", "beta", "truncnorm", "gamma"):
|
|
128
|
+
try:
|
|
129
|
+
cand = Parametric(name).fit(sample)
|
|
130
|
+
stat = stats.kstest(sample, cand.dist.cdf).statistic
|
|
131
|
+
except Exception:
|
|
132
|
+
continue
|
|
133
|
+
if np.isfinite(stat) and stat < best_stat:
|
|
134
|
+
best, best_stat = cand, stat
|
|
135
|
+
if best is None:
|
|
136
|
+
return Empirical().fit(x)
|
|
137
|
+
return best.fit(x)
|
|
138
|
+
if distribution not in DISTRIBUTIONS:
|
|
139
|
+
raise ValueError(f"Unknown distribution {distribution!r}; choose from {DISTRIBUTIONS}")
|
|
140
|
+
try:
|
|
141
|
+
return Parametric(distribution).fit(x)
|
|
142
|
+
except Exception:
|
|
143
|
+
return Empirical().fit(x)
|