synthschema 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,456 @@
1
+ """Value generators for columns that are never learned: keys, PII and free text.
2
+
3
+ Real values of these columns never reach a model and are never copied to the output.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import re
9
+ from collections.abc import Iterable
10
+
11
+ import numpy as np
12
+ import pandas as pd
13
+
14
+ from .metadata import Column
15
+
16
+ # ---------------------------------------------------------------------- regex patterns
17
+
18
+ _CLASS_CHARS = {
19
+ "0-9": "0123456789",
20
+ "A-Z": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
21
+ "a-z": "abcdefghijklmnopqrstuvwxyz",
22
+ }
23
+ _TOKEN_RE = re.compile(r"\[([^\]]+)\]|\\d|\\w|\\(.)|(.)")
24
+ _QUANT_RE = re.compile(r"\{(\d+)(?:,(\d+))?\}")
25
+
26
+
27
+ def _expand_class(spec: str) -> str:
28
+ chars: list[str] = []
29
+ i = 0
30
+ while i < len(spec):
31
+ if i + 2 < len(spec) and spec[i + 1] == "-":
32
+ start, end = ord(spec[i]), ord(spec[i + 2])
33
+ chars.extend(chr(c) for c in range(start, end + 1))
34
+ i += 3
35
+ else:
36
+ chars.append(spec[i])
37
+ i += 1
38
+ return "".join(dict.fromkeys(chars))
39
+
40
+
41
+ def parse_regex(pattern: str) -> list[tuple[str, int, int]]:
42
+ """Parse the small regex subset synthschema generates from.
43
+
44
+ Supported: literals, escaped literals, ``\\d``, ``\\w``, character classes such as
45
+ ``[A-Z]`` or ``[A-Za-z0-9_]``, and the quantifiers ``{n}`` and ``{m,n}``.
46
+ """
47
+ tokens: list[tuple[str, int, int]] = []
48
+ pos = 0
49
+ pattern = pattern.strip("^$")
50
+ while pos < len(pattern):
51
+ m = _TOKEN_RE.match(pattern, pos)
52
+ if m is None: # pragma: no cover - the token regex matches any character
53
+ raise ValueError(f"Cannot parse regex {pattern!r} at {pos}")
54
+ text = m.group(0)
55
+ if m.group(1) is not None:
56
+ chars = _expand_class(m.group(1))
57
+ elif text == r"\d":
58
+ chars = _CLASS_CHARS["0-9"]
59
+ elif text == r"\w":
60
+ chars = _CLASS_CHARS["A-Z"] + _CLASS_CHARS["a-z"] + _CLASS_CHARS["0-9"] + "_"
61
+ elif m.group(2) is not None:
62
+ chars = m.group(2)
63
+ else:
64
+ chars = m.group(3)
65
+ pos = m.end()
66
+ lo = hi = 1
67
+ q = _QUANT_RE.match(pattern, pos)
68
+ if q:
69
+ lo = int(q.group(1))
70
+ hi = int(q.group(2)) if q.group(2) else lo
71
+ pos = q.end()
72
+ tokens.append((chars, lo, hi))
73
+ return tokens
74
+
75
+
76
+ def _capacity(tokens: list[tuple[str, int, int]]) -> float:
77
+ cap = 1.0
78
+ for chars, _lo, hi in tokens:
79
+ cap *= float(len(chars)) ** hi
80
+ if cap > 1e18:
81
+ return cap
82
+ return cap
83
+
84
+
85
+ def _widen(tokens: list[tuple[str, int, int]], needed: int) -> list[tuple[str, int, int]]:
86
+ """Grow the last variable token until the pattern can hold ``needed`` distinct values."""
87
+ tokens = list(tokens)
88
+ variable = [i for i, (c, _, _) in enumerate(tokens) if len(c) > 1]
89
+ if not variable:
90
+ tokens.append((_CLASS_CHARS["0-9"], 1, 1))
91
+ variable = [len(tokens) - 1]
92
+ i = variable[-1]
93
+ while _capacity(tokens) < needed:
94
+ chars, lo, hi = tokens[i]
95
+ tokens[i] = (chars, lo + 1, hi + 1)
96
+ return tokens
97
+
98
+
99
+ def _generate_tokens(tokens: list[tuple[str, int, int]], n: int, rng: np.random.Generator) -> np.ndarray:
100
+ if n == 0:
101
+ return np.array([], dtype=object)
102
+ if all(lo == hi for _, lo, hi in tokens):
103
+ parts = []
104
+ for chars, k, _ in tokens:
105
+ if k == 0:
106
+ continue
107
+ pool = np.array(list(chars))
108
+ parts.append(pool[rng.integers(0, len(pool), size=(n, k))])
109
+ if not parts:
110
+ return np.full(n, "", dtype=object)
111
+ mat = np.ascontiguousarray(np.concatenate(parts, axis=1))
112
+ width = mat.shape[1]
113
+ return mat.view(f"<U{width}").ravel().astype(object)
114
+ out = []
115
+ for _ in range(n):
116
+ pieces = []
117
+ for chars, lo, hi in tokens:
118
+ k = int(rng.integers(lo, hi + 1))
119
+ pieces.append("".join(chars[j] for j in rng.integers(0, len(chars), size=k)))
120
+ out.append("".join(pieces))
121
+ return np.array(out, dtype=object)
122
+
123
+
124
+ def generate_from_regex(
125
+ pattern: str, n: int, rng: np.random.Generator, unique: bool = True
126
+ ) -> np.ndarray:
127
+ """Generate ``n`` strings that match ``pattern`` (unique by default)."""
128
+ tokens = parse_regex(pattern)
129
+ if not unique:
130
+ return _generate_tokens(tokens, n, rng)
131
+ if _capacity(tokens) < 2 * n:
132
+ tokens = _widen(tokens, 2 * n)
133
+ values = pd.unique(_generate_tokens(tokens, n, rng))
134
+ for _ in range(50):
135
+ if len(values) >= n:
136
+ break
137
+ extra = _generate_tokens(tokens, 2 * (n - len(values)) + 10, rng)
138
+ values = pd.unique(np.concatenate([values, extra]))
139
+ if len(values) < n: # pragma: no cover - only with tiny pattern spaces
140
+ raise ValueError(f"Could not generate {n} unique values for pattern {pattern!r}")
141
+ return np.asarray(values[:n], dtype=object)
142
+
143
+
144
+ def infer_regex(values: Iterable, max_samples: int = 2000) -> str | None:
145
+ """Learn a regex that describes a set of identifier strings, e.g. ``SKU-[0-9]{5}``."""
146
+ vals = [str(v) for v in values]
147
+ if not vals:
148
+ return None
149
+ if len(vals) > max_samples:
150
+ idx = np.random.default_rng(0).choice(len(vals), max_samples, replace=False)
151
+ vals = [vals[i] for i in idx]
152
+ lengths = {len(v) for v in vals}
153
+ if len(lengths) == 1:
154
+ length = lengths.pop()
155
+ position_chars = [{v[i] for v in vals} for i in range(length)]
156
+ # A run of digit positions is a number: if any digit in the run varies, the whole
157
+ # run is variable (so zero-padded counters such as 00042 become [0-9]{5}).
158
+ numeric_run = [False] * length
159
+ i = 0
160
+ while i < length:
161
+ if all(c.isdigit() for c in position_chars[i]):
162
+ j = i
163
+ while j < length and all(c.isdigit() for c in position_chars[j]):
164
+ j += 1
165
+ if any(len(position_chars[k]) > 1 for k in range(i, j)):
166
+ for k in range(i, j):
167
+ numeric_run[k] = True
168
+ i = j
169
+ else:
170
+ i += 1
171
+ tokens: list[str] = []
172
+ for i in range(length):
173
+ chars = position_chars[i]
174
+ if numeric_run[i]:
175
+ tokens.append("[0-9]")
176
+ elif len(chars) == 1:
177
+ tokens.append(re.escape(next(iter(chars))))
178
+ elif all(c.isdigit() for c in chars):
179
+ tokens.append("[0-9]")
180
+ elif all(c.isupper() and c.isalpha() for c in chars):
181
+ tokens.append("[A-Z]")
182
+ elif all(c.islower() and c.isalpha() for c in chars):
183
+ tokens.append("[a-z]")
184
+ elif all(c.isalnum() for c in chars):
185
+ tokens.append("[A-Za-z0-9]")
186
+ else:
187
+ return None
188
+ return _compress(tokens)
189
+ prefix = _common_prefix(vals)
190
+ rests = [v[len(prefix):] for v in vals]
191
+ width = max(len(r) for r in rests)
192
+ if all(r.isdigit() for r in rests if r):
193
+ return re.escape(prefix) + f"[0-9]{{{width}}}"
194
+ if all(r.isalnum() for r in rests if r):
195
+ return re.escape(prefix) + f"[A-Za-z0-9]{{{width}}}"
196
+ return None
197
+
198
+
199
+ def _compress(tokens: list[str]) -> str:
200
+ out: list[str] = []
201
+ i = 0
202
+ while i < len(tokens):
203
+ j = i
204
+ while j + 1 < len(tokens) and tokens[j + 1] == tokens[i]:
205
+ j += 1
206
+ run = j - i + 1
207
+ tok = tokens[i]
208
+ if tok.startswith("[") and run > 1:
209
+ out.append(f"{tok}{{{run}}}")
210
+ else:
211
+ out.append(tok * run)
212
+ i = j + 1
213
+ return "".join(out)
214
+
215
+
216
+ def _common_prefix(vals: list[str]) -> str:
217
+ prefix = vals[0]
218
+ for v in vals[1:]:
219
+ while not v.startswith(prefix):
220
+ prefix = prefix[:-1]
221
+ if not prefix:
222
+ return ""
223
+ # Do not swallow a varying digit run into the prefix.
224
+ while prefix and prefix[-1].isdigit():
225
+ prefix = prefix[:-1]
226
+ return prefix
227
+
228
+
229
+ # ---------------------------------------------------------------------- id generator
230
+
231
+
232
+ class IdGenerator:
233
+ """Generates key values that look like the real ones but never copy them."""
234
+
235
+ def __init__(self, column: Column | None = None) -> None:
236
+ self.column = column or Column("id")
237
+ self.kind = "int"
238
+ self.start = 1
239
+ self.pattern: str | None = self.column.regex_format
240
+ self.unique_ratio = 1.0
241
+ self.null_rate = 0.0
242
+
243
+ def fit(self, series: pd.Series) -> IdGenerator:
244
+ nonnull = series.dropna()
245
+ self.null_rate = float(series.isna().mean()) if len(series) else 0.0
246
+ self.unique_ratio = float(nonnull.nunique() / len(nonnull)) if len(nonnull) else 1.0
247
+ if self.column.representation == "int" or (
248
+ pd.api.types.is_numeric_dtype(series) and not pd.api.types.is_bool_dtype(series)
249
+ ):
250
+ self.kind = "int"
251
+ self.start = int(nonnull.min()) if len(nonnull) else 1
252
+ return self
253
+ if self.pattern is None and len(nonnull):
254
+ sample = nonnull.astype(str)
255
+ if sample.str.match(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$").all():
256
+ self.kind = "uuid"
257
+ return self
258
+ self.pattern = infer_regex(sample)
259
+ self.kind = "regex" if self.pattern else "uuid"
260
+ return self
261
+
262
+ def generate(
263
+ self, n: int, rng: np.random.Generator, unique: bool | None = None, allow_nulls: bool = True
264
+ ) -> np.ndarray:
265
+ if unique is None:
266
+ unique = self.unique_ratio >= 0.999
267
+ k = n if unique else max(1, int(round(self.unique_ratio * n)))
268
+ if self.kind == "int":
269
+ pool = np.arange(self.start, self.start + k, dtype=np.int64)
270
+ elif self.kind == "uuid":
271
+ raw = rng.integers(0, 16, size=(k, 32))
272
+ hexchars = np.array(list("0123456789abcdef"))[raw]
273
+ strings = np.ascontiguousarray(hexchars).view("<U32").ravel()
274
+ pool = np.array(
275
+ [f"{s[:8]}-{s[8:12]}-4{s[13:16]}-{s[16:20]}-{s[20:]}" for s in strings], dtype=object
276
+ )
277
+ else:
278
+ pool = generate_from_regex(self.pattern, k, rng, unique=True)
279
+ values = pool if unique else pool[rng.integers(0, len(pool), size=n)]
280
+ if allow_nulls and self.null_rate > 0:
281
+ values = values.astype(object)
282
+ values[rng.random(n) < self.null_rate] = None
283
+ return values
284
+
285
+
286
+ # ---------------------------------------------------------------------- PII and text
287
+
288
+ _FIRST = (
289
+ "James Mary Robert Patricia John Jennifer Michael Linda David Elizabeth William Barbara "
290
+ "Richard Susan Joseph Jessica Thomas Sarah Charles Karen Daniel Lisa Matthew Nancy Anthony "
291
+ "Sandra Mark Ashley Priya Arjun Wei Mei Omar Fatima Lucas Sofia Mateo Aisha Kenji Yuki"
292
+ ).split()
293
+ _LAST = (
294
+ "Smith Johnson Williams Brown Jones Garcia Miller Davis Rodriguez Martinez Hernandez Lopez "
295
+ "Gonzalez Wilson Anderson Thomas Taylor Moore Jackson Martin Lee Perez Thompson White Harris "
296
+ "Sanchez Clark Ramirez Lewis Robinson Patel Kumar Chen Wang Nguyen Kim Ali Khan Silva Rossi"
297
+ ).split()
298
+ _STREETS = "Main Oak Pine Maple Cedar Elm Washington Lake Hill Park River Sunset Highland".split()
299
+ _SUFFIX = "St Ave Rd Blvd Ln Dr Way Ct".split()
300
+ _CITIES = (
301
+ "Springfield Riverton Fairview Georgetown Franklin Greenville Bristol Clinton Madison "
302
+ "Salem Arlington Ashland Dover Hudson Kingston"
303
+ ).split()
304
+ _COMPANY_A = "Blue North Bright Prime Summit Silver Global Apex Pioneer Vertex Harbor Cedar".split()
305
+ _COMPANY_B = "Systems Labs Partners Holdings Analytics Logistics Foods Retail Health Energy".split()
306
+ _WORDS = (
307
+ "data quality order customer product service delivery price value report account team "
308
+ "support request update review payment store region market growth plan system process "
309
+ "issue result project network design contract offer return good fast late great poor "
310
+ "excellent average simple clear helpful friendly slow easy new standard premium"
311
+ ).split()
312
+
313
+ _FAKER_METHODS = {
314
+ "email": "email",
315
+ "phone_number": "phone_number",
316
+ "name": "name",
317
+ "first_name": "first_name",
318
+ "last_name": "last_name",
319
+ "address": "address",
320
+ "street_address": "street_address",
321
+ "ssn": "ssn",
322
+ "credit_card_number": "credit_card_number",
323
+ "ip_address": "ipv4_private",
324
+ "url": "url",
325
+ "company": "company",
326
+ "postcode": "postcode",
327
+ }
328
+
329
+ _FAKER_MAX_DIRECT = 100_000
330
+
331
+
332
+ def _faker(locale: str | list[str] | None, rng: np.random.Generator):
333
+ try:
334
+ from faker import Faker
335
+ except ImportError:
336
+ return None
337
+ fake = Faker(locale)
338
+ fake.seed_instance(int(rng.integers(0, 2**31 - 1)))
339
+ return fake
340
+
341
+
342
+ def _pick(pool: list[str], n: int, rng: np.random.Generator) -> np.ndarray:
343
+ return np.array(pool, dtype=object)[rng.integers(0, len(pool), size=n)]
344
+
345
+
346
+ def _digits(n: int, k: int, rng: np.random.Generator) -> np.ndarray:
347
+ mat = np.array(list("0123456789"))[rng.integers(0, 10, size=(n, k))]
348
+ return np.ascontiguousarray(mat).view(f"<U{k}").ravel().astype(object)
349
+
350
+
351
+ def _builtin_pii(sdtype: str, n: int, rng: np.random.Generator) -> np.ndarray:
352
+ first, last = _pick(_FIRST, n, rng), _pick(_LAST, n, rng)
353
+ if sdtype == "first_name":
354
+ return first
355
+ if sdtype == "last_name":
356
+ return last
357
+ if sdtype == "name":
358
+ return first + " " + last
359
+ if sdtype == "email":
360
+ num = _digits(n, 4, rng)
361
+ return np.char.lower((first + "." + last + num).astype(str)).astype(object) + "@example.com"
362
+ if sdtype == "phone_number":
363
+ return "+1-555-" + _digits(n, 3, rng) + "-" + _digits(n, 4, rng)
364
+ if sdtype in {"address", "street_address"}:
365
+ street = (
366
+ rng.integers(1, 9999, size=n).astype(str).astype(object)
367
+ + " " + _pick(_STREETS, n, rng) + " " + _pick(_SUFFIX, n, rng)
368
+ )
369
+ if sdtype == "street_address":
370
+ return street
371
+ return street + ", " + _pick(_CITIES, n, rng) + " " + _digits(n, 5, rng)
372
+ if sdtype == "ssn": # 9xx area numbers are never issued
373
+ return "9" + _digits(n, 2, rng) + "-" + _digits(n, 2, rng) + "-" + _digits(n, 4, rng)
374
+ if sdtype == "credit_card_number": # 4000 0000 prefix is a well-known test range
375
+ return "40000000" + _digits(n, 8, rng)
376
+ if sdtype == "ip_address":
377
+ octets = rng.integers(0, 256, size=(n, 3)).astype(str)
378
+ return np.array(["10." + ".".join(r) for r in octets], dtype=object)
379
+ if sdtype == "url":
380
+ return "https://www." + np.char.lower(_pick(_COMPANY_A, n, rng).astype(str)).astype(object) + _digits(n, 3, rng) + ".example.com"
381
+ if sdtype == "company":
382
+ return _pick(_COMPANY_A, n, rng) + " " + _pick(_COMPANY_B, n, rng)
383
+ if sdtype == "postcode":
384
+ return _digits(n, 5, rng)
385
+ raise ValueError(f"No generator for sdtype {sdtype!r}")
386
+
387
+
388
+ def generate_pii(
389
+ sdtype: str, n: int, rng: np.random.Generator, locale: str | list[str] | None = None
390
+ ) -> np.ndarray:
391
+ """Generate ``n`` fake values of a PII type. Uses Faker when installed."""
392
+ if n == 0:
393
+ return np.array([], dtype=object)
394
+ fake = _faker(locale, rng)
395
+ if fake is None:
396
+ return _builtin_pii(sdtype, n, rng)
397
+ method = getattr(fake, _FAKER_METHODS[sdtype])
398
+ size = min(n, _FAKER_MAX_DIRECT)
399
+ values = np.array([method() for _ in range(size)], dtype=object)
400
+ if sdtype == "address":
401
+ values = np.array([v.replace("\n", ", ") for v in values], dtype=object)
402
+ if size < n:
403
+ values = values[rng.integers(0, size, size=n)]
404
+ return values
405
+
406
+
407
+ class PiiGenerator:
408
+ def __init__(self, sdtype: str, locale: str | list[str] | None = None) -> None:
409
+ self.sdtype = sdtype
410
+ self.locale = locale
411
+ self.null_rate = 0.0
412
+
413
+ def fit(self, series: pd.Series) -> PiiGenerator:
414
+ self.null_rate = float(series.isna().mean()) if len(series) else 0.0
415
+ return self
416
+
417
+ def generate(self, n: int, rng: np.random.Generator) -> np.ndarray:
418
+ values = generate_pii(self.sdtype, n, rng, self.locale).astype(object)
419
+ if self.null_rate > 0:
420
+ values[rng.random(n) < self.null_rate] = None
421
+ return values
422
+
423
+
424
+ class TextGenerator:
425
+ """Generates placeholder free text with a word-count profile like the real column."""
426
+
427
+ def __init__(self, locale: str | list[str] | None = None) -> None:
428
+ self.locale = locale
429
+ self.mean_words = 8.0
430
+ self.std_words = 3.0
431
+ self.null_rate = 0.0
432
+
433
+ def fit(self, series: pd.Series) -> TextGenerator:
434
+ self.null_rate = float(series.isna().mean()) if len(series) else 0.0
435
+ counts = series.dropna().astype(str).str.split().str.len()
436
+ if len(counts):
437
+ self.mean_words = float(max(counts.mean(), 1))
438
+ self.std_words = float(counts.std() if len(counts) > 1 else 0) or 1.0
439
+ return self
440
+
441
+ def generate(self, n: int, rng: np.random.Generator) -> np.ndarray:
442
+ lengths = np.clip(np.round(rng.normal(self.mean_words, self.std_words, n)), 1, 200).astype(int)
443
+ fake = _faker(self.locale, rng)
444
+ if fake is not None and n <= _FAKER_MAX_DIRECT:
445
+ values = np.array(
446
+ [fake.sentence(nb_words=int(k), variable_nb_words=False) for k in lengths], dtype=object
447
+ )
448
+ else:
449
+ words = np.array(_WORDS, dtype=object)
450
+ values = np.array(
451
+ [" ".join(words[rng.integers(0, len(words), size=k)]).capitalize() + "." for k in lengths],
452
+ dtype=object,
453
+ )
454
+ if self.null_rate > 0:
455
+ values[rng.random(n) < self.null_rate] = None
456
+ return values
@@ -0,0 +1,143 @@
1
+ """Univariate (marginal) distributions used by the Gaussian copula."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import warnings
6
+
7
+ import numpy as np
8
+ from scipy import stats
9
+
10
+ EPS = 1e-6
11
+ _MAX_EMPIRICAL_POINTS = 50_000
12
+
13
+ DISTRIBUTIONS = ("empirical", "norm", "uniform", "beta", "truncnorm", "gamma", "auto")
14
+
15
+
16
+ class Marginal:
17
+ name = "base"
18
+
19
+ def fit(self, x: np.ndarray) -> Marginal: # pragma: no cover - interface
20
+ raise NotImplementedError
21
+
22
+ def cdf(self, x: np.ndarray, rng: np.random.Generator | None = None) -> np.ndarray: # pragma: no cover
23
+ raise NotImplementedError
24
+
25
+ def ppf(self, u: np.ndarray) -> np.ndarray: # pragma: no cover
26
+ raise NotImplementedError
27
+
28
+
29
+ class Constant(Marginal):
30
+ name = "constant"
31
+
32
+ def fit(self, x):
33
+ self.value = float(x[0]) if len(x) else 0.0
34
+ return self
35
+
36
+ def cdf(self, x, rng=None):
37
+ return np.full(len(x), 0.5)
38
+
39
+ def ppf(self, u):
40
+ return np.full(len(u), self.value)
41
+
42
+
43
+ class Empirical(Marginal):
44
+ """Exact empirical distribution with linear interpolation between observed quantiles.
45
+
46
+ Ties (discrete values) are spread uniformly over their probability mass so the
47
+ probability-integral transform stays uniform.
48
+ """
49
+
50
+ name = "empirical"
51
+
52
+ def fit(self, x):
53
+ x = np.sort(np.asarray(x, dtype=float))
54
+ if len(x) > _MAX_EMPIRICAL_POINTS:
55
+ # x is sorted, so linear interpolation over ranks equals np.quantile (and is O(n)).
56
+ positions = np.linspace(0, len(x) - 1, _MAX_EMPIRICAL_POINTS)
57
+ x = np.interp(positions, np.arange(len(x)), x)
58
+ self.sorted = x
59
+ n = len(x)
60
+ self.probs = (np.arange(n) + 0.5) / n
61
+ return self
62
+
63
+ def cdf(self, x, rng=None):
64
+ x = np.asarray(x, dtype=float)
65
+ s, n = self.sorted, len(self.sorted)
66
+ left = np.searchsorted(s, x, side="left")
67
+ right = np.searchsorted(s, x, side="right")
68
+ tied = right > left
69
+ u = np.interp(x, s, self.probs)
70
+ if tied.any():
71
+ frac = rng.random(tied.sum()) if rng is not None else 0.5
72
+ u[tied] = (left[tied] + frac * (right[tied] - left[tied])) / n
73
+ return np.clip(u, EPS, 1 - EPS)
74
+
75
+ def ppf(self, u):
76
+ return np.interp(np.asarray(u, dtype=float), self.probs, self.sorted)
77
+
78
+
79
+ class Parametric(Marginal):
80
+ """A scipy.stats distribution fitted by maximum likelihood."""
81
+
82
+ def __init__(self, name: str) -> None:
83
+ self.name = name
84
+
85
+ def fit(self, x):
86
+ x = np.asarray(x, dtype=float)
87
+ lo, hi = float(x.min()), float(x.max())
88
+ span = hi - lo
89
+ with warnings.catch_warnings():
90
+ warnings.simplefilter("ignore")
91
+ if self.name == "norm":
92
+ self.dist = stats.norm(*stats.norm.fit(x))
93
+ elif self.name == "uniform":
94
+ self.dist = stats.uniform(lo, span)
95
+ elif self.name == "beta":
96
+ loc, scale = lo - 1e-3 * span, span * 1.002
97
+ a, b, _, _ = stats.beta.fit(x, floc=loc, fscale=scale)
98
+ self.dist = stats.beta(a, b, loc=loc, scale=scale)
99
+ elif self.name == "truncnorm":
100
+ mu, sd = float(x.mean()), float(x.std()) or 1.0
101
+ self.dist = stats.truncnorm((lo - mu) / sd, (hi - mu) / sd, loc=mu, scale=sd)
102
+ elif self.name == "gamma":
103
+ a, loc, scale = stats.gamma.fit(x, floc=lo - 1e-3 * span)
104
+ self.dist = stats.gamma(a, loc=loc, scale=scale)
105
+ else:
106
+ raise ValueError(f"Unknown distribution {self.name!r}")
107
+ return self
108
+
109
+ def cdf(self, x, rng=None):
110
+ return np.clip(self.dist.cdf(np.asarray(x, dtype=float)), EPS, 1 - EPS)
111
+
112
+ def ppf(self, u):
113
+ return self.dist.ppf(np.asarray(u, dtype=float))
114
+
115
+
116
+ def fit_marginal(x: np.ndarray, distribution: str = "empirical") -> Marginal:
117
+ """Fit a marginal. ``"auto"`` picks the parametric family with the best KS statistic."""
118
+ x = np.asarray(x, dtype=float)
119
+ x = x[np.isfinite(x)]
120
+ if len(x) == 0 or np.all(x == x[0]):
121
+ return Constant().fit(x)
122
+ if distribution == "empirical":
123
+ return Empirical().fit(x)
124
+ if distribution == "auto":
125
+ sample = x if len(x) <= 5000 else np.random.default_rng(0).choice(x, 5000, replace=False)
126
+ best, best_stat = None, np.inf
127
+ for name in ("norm", "uniform", "beta", "truncnorm", "gamma"):
128
+ try:
129
+ cand = Parametric(name).fit(sample)
130
+ stat = stats.kstest(sample, cand.dist.cdf).statistic
131
+ except Exception:
132
+ continue
133
+ if np.isfinite(stat) and stat < best_stat:
134
+ best, best_stat = cand, stat
135
+ if best is None:
136
+ return Empirical().fit(x)
137
+ return best.fit(x)
138
+ if distribution not in DISTRIBUTIONS:
139
+ raise ValueError(f"Unknown distribution {distribution!r}; choose from {DISTRIBUTIONS}")
140
+ try:
141
+ return Parametric(distribution).fit(x)
142
+ except Exception:
143
+ return Empirical().fit(x)