model2data 1.6.0__tar.gz → 1.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {model2data-1.6.0/model2data.egg-info → model2data-1.7.1}/PKG-INFO +1 -1
- {model2data-1.6.0 → model2data-1.7.1}/README.md +24 -1
- {model2data-1.6.0 → model2data-1.7.1}/model2data/generate/core.py +33 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/generate/faker.py +176 -10
- {model2data-1.6.0 → model2data-1.7.1}/model2data/generate/hints.py +73 -4
- {model2data-1.6.0 → model2data-1.7.1/model2data.egg-info}/PKG-INFO +1 -1
- {model2data-1.6.0 → model2data-1.7.1}/model2data.egg-info/SOURCES.txt +2 -0
- {model2data-1.6.0 → model2data-1.7.1}/pyproject.toml +1 -1
- model2data-1.7.1/tests/test_distributions.py +258 -0
- model2data-1.7.1/tests/test_lone_country.py +132 -0
- {model2data-1.6.0 → model2data-1.7.1}/LICENSE +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/README_PYPI.md +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/__init__.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/cli.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/dbt/__init__.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/dbt/project.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/dbt/templates/dbt_project.yml.jinja +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/dbt/templates/macros/generate_schema_name.sql +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/dbt/templates/profiles.yml.jinja +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/dbt/tests.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/generate/__init__.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/generate/options.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/generate/relationships.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/generate/timeline.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/parse/__init__.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/parse/dbml.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data/utils.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data.egg-info/dependency_links.txt +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data.egg-info/entry_points.txt +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data.egg-info/requires.txt +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/model2data.egg-info/top_level.txt +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/setup.cfg +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_as_of_anchor.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_cli.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_column_time_hints.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_coverage_gaps.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_dbml_parser.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_dbml_parser_fuzz.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_dbt_integration.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_dbt_naming.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_dbt_project.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_dbt_tests.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_faker_name_inference.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_generation.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_options.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_release_stress.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_row_identity.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_shaping.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_table_seeds.py +0 -0
- {model2data-1.6.0 → model2data-1.7.1}/tests/test_timeline.py +0 -0
|
@@ -159,7 +159,10 @@ model2data --file examples/ecommerce.dbml --rows 200 --seed 42 --table-seed orde
|
|
|
159
159
|
```
|
|
160
160
|
|
|
161
161
|
`--locale` picks the country every generated person and address comes from (`en_US` by default);
|
|
162
|
-
it's a per-run setting, so a table can't end up holding one Belgian and one American address
|
|
162
|
+
it's a per-run setting, so a table can't end up holding one Belgian and one American address. A
|
|
163
|
+
`country` column that sits beside a `city`/`street`/`state`/`postcode` column always agrees with
|
|
164
|
+
that place; a `country` column with none of those beside it isn't describing anyone's address, so
|
|
165
|
+
it reads as an international mix instead, with the locale's own country the most common:
|
|
163
166
|
|
|
164
167
|
```bash
|
|
165
168
|
model2data --file examples/ecommerce.dbml --rows 200 --seed 42 --locale nl_BE
|
|
@@ -232,6 +235,26 @@ non-null rows a boolean column comes back `true`. `null_rate` replaces the colum
|
|
|
232
235
|
fraction outright. `distinct` draws the column's values from a fixed-size pool instead of a fresh
|
|
233
236
|
value per row — a `shipping_city` most warehouses only ever see a handful of.
|
|
234
237
|
|
|
238
|
+
### Shape a number's distribution
|
|
239
|
+
|
|
240
|
+
`min`/`max` alone only ever drew uniformly between them. A `distribution` note hint on an
|
|
241
|
+
integer or decimal column picks a different shape instead:
|
|
242
|
+
|
|
243
|
+
```dbml
|
|
244
|
+
Table orders {
|
|
245
|
+
id int [pk]
|
|
246
|
+
total_amount numeric [note: '{"distribution": "lognormal", "median": 80, "spread": 0.6, "min": 5}']
|
|
247
|
+
}
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
`normal` takes `mean` (the centre) and `stddev` (the spread); `lognormal` takes `median` (the
|
|
251
|
+
typical value) and `spread` (how heavy the tail is — 0.3 is mild, 1.0 is heavy); `exponential`
|
|
252
|
+
takes `mean` (the average). Any left unset default to the midpoint of the column's effective
|
|
253
|
+
`min`/`max` (or `stddev` = range / 6, `spread` = 0.5). `min`/`max` still clip the result — a
|
|
254
|
+
`normal` centred near an edge redraws a bounded number of times before clamping, so it never
|
|
255
|
+
loops forever and never crosses the bound. Leaving `distribution` out, or setting it to
|
|
256
|
+
`"uniform"`, is exactly today's behaviour.
|
|
257
|
+
|
|
235
258
|
Run dbt to load, transform, and test the data:
|
|
236
259
|
|
|
237
260
|
```bash
|
|
@@ -15,6 +15,7 @@ from model2data.generate.faker import (
|
|
|
15
15
|
release_row_pools,
|
|
16
16
|
reset_duplicate_unique_columns,
|
|
17
17
|
reset_row_pools,
|
|
18
|
+
resolve_address_pool_field,
|
|
18
19
|
set_locale,
|
|
19
20
|
)
|
|
20
21
|
from model2data.generate.hints import validate_hints
|
|
@@ -201,6 +202,12 @@ def generate_data_from_dbml(
|
|
|
201
202
|
for column_name in key.get("columns") or []
|
|
202
203
|
}
|
|
203
204
|
|
|
205
|
+
# A table's own shape, worked out before a single value is drawn: does
|
|
206
|
+
# this table have a `country` column with no `city`/`street`/`state`/
|
|
207
|
+
# `postcode` beside it to keep coherent with. See
|
|
208
|
+
# _lone_country_columns.
|
|
209
|
+
lone_country_columns = _lone_country_columns(table_def)
|
|
210
|
+
|
|
204
211
|
# -----------------------
|
|
205
212
|
# First pass: columns + FKs
|
|
206
213
|
# -----------------------
|
|
@@ -230,6 +237,7 @@ def generate_data_from_dbml(
|
|
|
230
237
|
as_of=as_of,
|
|
231
238
|
time_profile=profile,
|
|
232
239
|
skew=skew,
|
|
240
|
+
lone_country=column.name in lone_country_columns,
|
|
233
241
|
)
|
|
234
242
|
|
|
235
243
|
df = pd.DataFrame(data)
|
|
@@ -303,6 +311,31 @@ def generate_data_from_dbml(
|
|
|
303
311
|
# ---------------------------------------------------------
|
|
304
312
|
# Internal helpers
|
|
305
313
|
# ---------------------------------------------------------
|
|
314
|
+
def _lone_country_columns(table_def: TableDef) -> set[str]:
|
|
315
|
+
"""Names of this table's *lone* country columns.
|
|
316
|
+
|
|
317
|
+
A `country` column reads as the locale's own country on every row when it
|
|
318
|
+
sits beside a `city`/`street`/`state`/`postcode` column -- together they
|
|
319
|
+
describe one place, and the country has to agree with the rest of it.
|
|
320
|
+
Alone, repeating that same country on every row reads as a single-country
|
|
321
|
+
customer base rather than an international one, so
|
|
322
|
+
`generate_column_values` draws it from a home-heavy mix instead (see
|
|
323
|
+
`faker._HOME_COUNTRY_SHARE`). `country` columns don't count as company
|
|
324
|
+
for each other -- only a *different* address-pool field does.
|
|
325
|
+
"""
|
|
326
|
+
address_fields = {
|
|
327
|
+
column.name: field
|
|
328
|
+
for column in table_def.columns
|
|
329
|
+
for field in [resolve_address_pool_field(column)]
|
|
330
|
+
if field is not None
|
|
331
|
+
}
|
|
332
|
+
country_columns = {name for name, field in address_fields.items() if field == "country"}
|
|
333
|
+
if not country_columns:
|
|
334
|
+
return set()
|
|
335
|
+
has_place_column = any(field != "country" for field in address_fields.values())
|
|
336
|
+
return set() if has_place_column else country_columns
|
|
337
|
+
|
|
338
|
+
|
|
306
339
|
def _coerce_integer_dtypes(df: pd.DataFrame, table_def: TableDef) -> pd.DataFrame:
|
|
307
340
|
"""
|
|
308
341
|
Cast int/bigint/smallint-typed columns to pandas' nullable "Int64" dtype.
|
|
@@ -505,6 +505,42 @@ def _infer_by_type(base_type: str) -> Optional[_Provider]:
|
|
|
505
505
|
return lambda: fake.format(base_type)
|
|
506
506
|
|
|
507
507
|
|
|
508
|
+
def resolve_address_pool_field(column: ColumnDef) -> Optional[str]:
|
|
509
|
+
"""The address-pool field (`street`, `full`, `city`, `state`, `postcode`,
|
|
510
|
+
`country`) this column would draw from, if any -- else None.
|
|
511
|
+
|
|
512
|
+
Same declared-type-then-name precedence `generate_column_values`'s own
|
|
513
|
+
untyped-column branch uses (`_infer_by_type` before `_infer_by_name`),
|
|
514
|
+
narrowed to the address pool. Exposed for `generate.core` to tell a
|
|
515
|
+
*lone* country column -- the only address-pool-shaped column in its
|
|
516
|
+
table -- from one that sits beside a `city`/`street`/`state`/`postcode`
|
|
517
|
+
column, before a single value of the table has been generated.
|
|
518
|
+
|
|
519
|
+
Restricted to columns that would actually reach that branch: an enum
|
|
520
|
+
column, or one whose type is a structured (int/date/uuid/...) type, never
|
|
521
|
+
gets there in `generate_column_values` -- and `_infer_by_type` probes an
|
|
522
|
+
unrecognized type by actually calling it (`fake.format(base_type)`), so
|
|
523
|
+
running it over a `date` or `int` column here would consume real draws
|
|
524
|
+
from the shared RNG and shift every value generated after it, breaking
|
|
525
|
+
reproducibility for reasons invisible to whoever hits it.
|
|
526
|
+
"""
|
|
527
|
+
if column.enum_values or not is_free_text_type(column.data_type):
|
|
528
|
+
return None
|
|
529
|
+
base_type = column.data_type.lower().split("(")[0].strip()
|
|
530
|
+
generator = _infer_by_type(base_type) or _infer_by_name(column.name)
|
|
531
|
+
if isinstance(generator, _FromRow) and generator.pool == "address":
|
|
532
|
+
return generator.field
|
|
533
|
+
return None
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
# A lone country column (see resolve_address_pool_field's caller) mixes the
|
|
537
|
+
# locale's own country in with the rest of the world rather than repeating it
|
|
538
|
+
# on every row. 0.6 is a default, not a claim about any real market -- a
|
|
539
|
+
# business selling internationally still has a home market, and most of its
|
|
540
|
+
# rows are plausibly it, but "most" is not "all".
|
|
541
|
+
_HOME_COUNTRY_SHARE = 0.6
|
|
542
|
+
|
|
543
|
+
|
|
508
544
|
def _column_time_profile(
|
|
509
545
|
column: ColumnDef, time_profile: Optional[TimeProfile]
|
|
510
546
|
) -> Optional[TimeProfile]:
|
|
@@ -580,6 +616,7 @@ def generate_column_values(
|
|
|
580
616
|
as_of: AsOf = None,
|
|
581
617
|
time_profile: Optional[TimeProfile] = None,
|
|
582
618
|
skew: float = 0.0,
|
|
619
|
+
lone_country: bool = False,
|
|
583
620
|
) -> list:
|
|
584
621
|
"""
|
|
585
622
|
Generate synthetic values for a single column.
|
|
@@ -591,6 +628,16 @@ def generate_column_values(
|
|
|
591
628
|
every path that draws a value -- the main pass, the self-referencing FK
|
|
592
629
|
repair, the composite-key retry -- draws it the same way.
|
|
593
630
|
|
|
631
|
+
`lone_country` tells the address-pool branch this column is the *only*
|
|
632
|
+
address-shaped column in its table (see
|
|
633
|
+
`generate.core._lone_country_columns`). A `country` column that sits
|
|
634
|
+
beside a `city`/`street`/`state`/`postcode` column still reads that
|
|
635
|
+
place's own country, byte-identical to earlier releases; a lone one
|
|
636
|
+
instead draws a home-heavy mix of the locale's country and the wider
|
|
637
|
+
world, since "Belgium" on every row of a customers table with no other
|
|
638
|
+
address column reads as a single-country customer base rather than an
|
|
639
|
+
international one.
|
|
640
|
+
|
|
594
641
|
`as_of` is the date every generated date and timestamp is placed relative
|
|
595
642
|
to, defaulting to today. Pass it to make a seeded run reproduce on any
|
|
596
643
|
later day rather than only on the day it first ran.
|
|
@@ -629,6 +676,7 @@ def generate_column_values(
|
|
|
629
676
|
as_of=as_of,
|
|
630
677
|
time_profile=time_profile,
|
|
631
678
|
skew=skew,
|
|
679
|
+
lone_country=lone_country,
|
|
632
680
|
)
|
|
633
681
|
values = random.choices(pool, k=row_count)
|
|
634
682
|
if not force_not_null and "not null" not in column.settings and "pk" not in column.settings:
|
|
@@ -665,6 +713,13 @@ def generate_column_values(
|
|
|
665
713
|
if column.note:
|
|
666
714
|
min_val = column.note.get("min")
|
|
667
715
|
max_val = column.note.get("max")
|
|
716
|
+
# The note's own min/max, before either numeric branch fills in its
|
|
717
|
+
# branch-specific default -- these are the only bounds a distribution
|
|
718
|
+
# hint clips against (see `_clipped_distribution_draw`). A column with no
|
|
719
|
+
# explicit min/max draws from an unclipped distribution even though the
|
|
720
|
+
# branch below still has an implicit default range for the *parameters*.
|
|
721
|
+
explicit_min, explicit_max = min_val, max_val
|
|
722
|
+
distribution = column_note.get("distribution", "uniform")
|
|
668
723
|
|
|
669
724
|
if fk_series is not None and not fk_series.empty:
|
|
670
725
|
# A plain branch of the same if/elif chain (rather than an early
|
|
@@ -722,7 +777,22 @@ def generate_column_values(
|
|
|
722
777
|
if max_val is None:
|
|
723
778
|
max_val = 100
|
|
724
779
|
|
|
725
|
-
if
|
|
780
|
+
if distribution != "uniform":
|
|
781
|
+
# A shaped column never needs the row_count-sized value space
|
|
782
|
+
# random.sample relies on below, so it always draws one value at
|
|
783
|
+
# a time -- unique or not -- and leans on _deduplicate the same
|
|
784
|
+
# way every other non-uniform-fast-path branch does.
|
|
785
|
+
params = _distribution_params(distribution, min_val, max_val, column_note)
|
|
786
|
+
|
|
787
|
+
def generator() -> int:
|
|
788
|
+
return round(
|
|
789
|
+
_clipped_distribution_draw(distribution, params, explicit_min, explicit_max)
|
|
790
|
+
)
|
|
791
|
+
|
|
792
|
+
values = [generator() for _ in range(row_count)]
|
|
793
|
+
if ensure_unique:
|
|
794
|
+
values = _deduplicate(values, generator, column_name=unique_label)
|
|
795
|
+
elif ensure_unique:
|
|
726
796
|
if not had_explicit_range:
|
|
727
797
|
# No user-specified range: widen the default so there's
|
|
728
798
|
# always enough headroom for `row_count` unique PK values.
|
|
@@ -753,13 +823,27 @@ def generate_column_values(
|
|
|
753
823
|
min_val = 0
|
|
754
824
|
if max_val is None:
|
|
755
825
|
max_val = 10_000
|
|
756
|
-
|
|
757
|
-
if
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
826
|
+
|
|
827
|
+
if distribution != "uniform":
|
|
828
|
+
params = _distribution_params(distribution, min_val, max_val, column_note)
|
|
829
|
+
|
|
830
|
+
def generator() -> float:
|
|
831
|
+
return round(
|
|
832
|
+
_clipped_distribution_draw(distribution, params, explicit_min, explicit_max),
|
|
833
|
+
2,
|
|
834
|
+
)
|
|
835
|
+
|
|
836
|
+
values = [generator() for _ in range(row_count)]
|
|
837
|
+
if ensure_unique:
|
|
838
|
+
values = _deduplicate(values, generator, column_name=unique_label)
|
|
839
|
+
else:
|
|
840
|
+
values = [round(random.uniform(min_val, max_val), 2) for _ in range(row_count)]
|
|
841
|
+
if ensure_unique:
|
|
842
|
+
values = _deduplicate(
|
|
843
|
+
values,
|
|
844
|
+
lambda: round(random.uniform(min_val, max_val), 2),
|
|
845
|
+
column_name=unique_label,
|
|
846
|
+
)
|
|
763
847
|
|
|
764
848
|
# -----------------------------------------------------
|
|
765
849
|
# Booleans
|
|
@@ -791,8 +875,18 @@ def generate_column_values(
|
|
|
791
875
|
else:
|
|
792
876
|
generator = _infer_by_type(base_type) or _infer_by_name(column.name)
|
|
793
877
|
if isinstance(generator, _FromRow):
|
|
794
|
-
|
|
795
|
-
|
|
878
|
+
if lone_country and generator.pool == "address" and generator.field == "country":
|
|
879
|
+
# No sibling city/street/state/postcode column to keep this
|
|
880
|
+
# one coherent with, so it isn't "this row's place" at all --
|
|
881
|
+
# draw a home-heavy mix instead of repeating the locale's own
|
|
882
|
+
# country on every row.
|
|
883
|
+
values = [
|
|
884
|
+
_country_name if random.random() < _HOME_COUNTRY_SHARE else fake.country()
|
|
885
|
+
for _ in range(row_count)
|
|
886
|
+
]
|
|
887
|
+
else:
|
|
888
|
+
rows = _row_pool(generator.pool, table_name, row_count)
|
|
889
|
+
values = [getattr(rows[index], generator.field) for index in range(row_count)]
|
|
796
890
|
if ensure_unique:
|
|
797
891
|
values = _deduplicate_identity(values)
|
|
798
892
|
elif generator is not None:
|
|
@@ -824,6 +918,78 @@ def generate_column_values(
|
|
|
824
918
|
# ---------------------------------------------------------
|
|
825
919
|
# Internal helpers
|
|
826
920
|
# ---------------------------------------------------------
|
|
921
|
+
# How many times a clipped draw redraws before giving up and clamping. Kept
|
|
922
|
+
# small and fixed rather than exposed as a hint: a normal whose mean sits
|
|
923
|
+
# well inside its min/max rarely needs a redraw at all, and a distribution
|
|
924
|
+
# whose parameters put most of its mass outside the bounds is a schema
|
|
925
|
+
# mistake no retry count fixes -- clamping is the honest fallback either way.
|
|
926
|
+
_MAX_DISTRIBUTION_REDRAWS = 20
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
def _distribution_params(
|
|
930
|
+
distribution: str, effective_min: float, effective_max: float, note: dict
|
|
931
|
+
) -> dict:
|
|
932
|
+
"""Fill in whichever of mean/stddev/median/spread `note` leaves unset.
|
|
933
|
+
|
|
934
|
+
`effective_min`/`effective_max` are the note's own `min`/`max` when given,
|
|
935
|
+
else the calling branch's default range (0-100 for integers, 0-10,000 for
|
|
936
|
+
decimals) -- ordinary enough a midpoint to centre an unparameterized
|
|
937
|
+
distribution on.
|
|
938
|
+
"""
|
|
939
|
+
midpoint = (effective_min + effective_max) / 2
|
|
940
|
+
if distribution == "normal":
|
|
941
|
+
return {
|
|
942
|
+
"mean": note.get("mean", midpoint),
|
|
943
|
+
"stddev": note.get("stddev", (effective_max - effective_min) / 6),
|
|
944
|
+
}
|
|
945
|
+
if distribution == "lognormal":
|
|
946
|
+
return {
|
|
947
|
+
"median": note.get("median", midpoint),
|
|
948
|
+
"spread": note.get("spread", 0.5),
|
|
949
|
+
}
|
|
950
|
+
# exponential: hints.py has already rejected anything else reaching here.
|
|
951
|
+
return {"mean": note.get("mean", midpoint)}
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
def _draw_distribution_value(distribution: str, params: dict) -> float:
|
|
955
|
+
"""One raw draw from `distribution`, unclipped and unrounded."""
|
|
956
|
+
if distribution == "normal":
|
|
957
|
+
return random.gauss(params["mean"], params["stddev"])
|
|
958
|
+
if distribution == "lognormal":
|
|
959
|
+
return random.lognormvariate(math.log(params["median"]), params["spread"])
|
|
960
|
+
return random.expovariate(1 / params["mean"]) # exponential
|
|
961
|
+
|
|
962
|
+
|
|
963
|
+
def _clipped_distribution_draw(
|
|
964
|
+
distribution: str,
|
|
965
|
+
params: dict,
|
|
966
|
+
clip_min: Optional[float],
|
|
967
|
+
clip_max: Optional[float],
|
|
968
|
+
) -> float:
|
|
969
|
+
"""Draw from `distribution`, redrawing out-of-bounds values before clamping.
|
|
970
|
+
|
|
971
|
+
A normal centred well inside [min, max] almost never needs the clamp; one
|
|
972
|
+
centred near an edge (mean 120, min 0) would otherwise pile values up at
|
|
973
|
+
the boundary, so a bounded number of redraws is tried first and only a
|
|
974
|
+
value still out of bounds after all of them gets clamped -- which also
|
|
975
|
+
keeps this from looping forever when min and max leave no room at all.
|
|
976
|
+
`clip_min`/`clip_max` are the note's own bounds (`None` when the column
|
|
977
|
+
didn't set one), never the branch's implicit default range.
|
|
978
|
+
"""
|
|
979
|
+
value = _draw_distribution_value(distribution, params)
|
|
980
|
+
attempts = 0
|
|
981
|
+
while attempts < _MAX_DISTRIBUTION_REDRAWS and (
|
|
982
|
+
(clip_min is not None and value < clip_min) or (clip_max is not None and value > clip_max)
|
|
983
|
+
):
|
|
984
|
+
value = _draw_distribution_value(distribution, params)
|
|
985
|
+
attempts += 1
|
|
986
|
+
if clip_min is not None and value < clip_min:
|
|
987
|
+
value = clip_min
|
|
988
|
+
if clip_max is not None and value > clip_max:
|
|
989
|
+
value = clip_max
|
|
990
|
+
return value
|
|
991
|
+
|
|
992
|
+
|
|
827
993
|
def _null_fraction_for(column: ColumnDef, row_count: int) -> float:
|
|
828
994
|
"""The fraction of `row_count` rows this column should turn null.
|
|
829
995
|
|
|
@@ -3,10 +3,11 @@
|
|
|
3
3
|
A note has always been either plain text (a comment, ignored by generation)
|
|
4
4
|
or a JSON object read for `min`/`max`. This module documents the rest of that
|
|
5
5
|
object's vocabulary -- `null_rate`, `weights`, `true_rate`, `distinct`, `skew`,
|
|
6
|
-
`after`, `business_hours`, `growth`, `seasonality`
|
|
7
|
-
before a single row is generated, so a
|
|
8
|
-
wrong kind of column fails with a message
|
|
9
|
-
than surfacing as a wrong-looking dataset
|
|
6
|
+
`after`, `business_hours`, `growth`, `seasonality`, `distribution` and its
|
|
7
|
+
parameters -- and checks it once, before a single row is generated, so a
|
|
8
|
+
typo'd enum value or a hint on the wrong kind of column fails with a message
|
|
9
|
+
naming the table and column rather than surfacing as a wrong-looking dataset
|
|
10
|
+
or a downstream dbt test failure.
|
|
10
11
|
|
|
11
12
|
| key | applies to | meaning |
|
|
12
13
|
|-------------|-----------------------------------------------|--------------------------------------------|
|
|
@@ -20,6 +21,11 @@ than surfacing as a wrong-looking dataset or a downstream dbt test failure.
|
|
|
20
21
|
| `business_hours` | date/timestamp columns | overrides the run-level `TimeProfile.business_hours` for this column |
|
|
21
22
|
| `growth` | date/timestamp columns | overrides the run-level `TimeProfile.growth` for this column |
|
|
22
23
|
| `seasonality` | date/timestamp columns | overrides the run-level `TimeProfile.seasonality` for this column |
|
|
24
|
+
| `distribution` | numeric columns | draw shape: `uniform` (default), `normal`, `lognormal`, `exponential` |
|
|
25
|
+
| `mean` | numeric columns, with `distribution: normal` or `exponential` | normal: the centre; exponential: the average (scale) |
|
|
26
|
+
| `stddev` | numeric columns, with `distribution: normal` | normal: the spread |
|
|
27
|
+
| `median` | numeric columns, with `distribution: lognormal`| lognormal: the typical value, `exp(mu)` |
|
|
28
|
+
| `spread` | numeric columns, with `distribution: lognormal`| lognormal: sigma of the underlying normal (0.3 mild, 1.0 heavy tail) |
|
|
23
29
|
"""
|
|
24
30
|
|
|
25
31
|
from __future__ import annotations
|
|
@@ -95,6 +101,33 @@ def _is_temporal_type(base_type: str) -> bool:
|
|
|
95
101
|
return any(key in base_type for key in ("timestamp", "datetime"))
|
|
96
102
|
|
|
97
103
|
|
|
104
|
+
def _is_numeric_type(base_type: str) -> bool:
|
|
105
|
+
"""True for the integer and float/decimal types `min`/`max`/`distribution` apply to.
|
|
106
|
+
|
|
107
|
+
Mirrors generate.faker's own two numeric branches exactly, so a hint this
|
|
108
|
+
module accepts is guaranteed to land on a branch that reads it.
|
|
109
|
+
"""
|
|
110
|
+
return any(
|
|
111
|
+
key in base_type
|
|
112
|
+
for key in ("int", "integer", "bigint", "smallint", "decimal", "numeric", "float", "double")
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# Distributions a numeric column's `distribution` hint may name.
|
|
117
|
+
_DISTRIBUTIONS = ("uniform", "normal", "lognormal", "exponential")
|
|
118
|
+
|
|
119
|
+
# Which distribution(s) each shape parameter is meaningful under -- checked
|
|
120
|
+
# against the note's own `distribution` (default "uniform" when absent), so
|
|
121
|
+
# a `stddev` with no `distribution` key is rejected the same as one paired
|
|
122
|
+
# with the wrong distribution.
|
|
123
|
+
_PARAM_DISTRIBUTIONS: dict[str, tuple[str, ...]] = {
|
|
124
|
+
"mean": ("normal", "exponential"),
|
|
125
|
+
"stddev": ("normal",),
|
|
126
|
+
"median": ("lognormal",),
|
|
127
|
+
"spread": ("lognormal",),
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
|
|
98
131
|
def _validate_column_hints(
|
|
99
132
|
table_name: str,
|
|
100
133
|
column: ColumnDef,
|
|
@@ -110,6 +143,7 @@ def _validate_column_hints(
|
|
|
110
143
|
is_enum = bool(column.enum_values)
|
|
111
144
|
is_boolean = "boolean" in base_type or "bool" in base_type
|
|
112
145
|
is_temporal = _is_temporal_type(base_type)
|
|
146
|
+
is_numeric = _is_numeric_type(base_type)
|
|
113
147
|
is_nullable = not is_pk and "not null" not in column.settings
|
|
114
148
|
|
|
115
149
|
if "null_rate" in note:
|
|
@@ -170,6 +204,31 @@ def _validate_column_hints(
|
|
|
170
204
|
raise ValueError(f'{label}: "seasonality" only applies to date/timestamp columns.')
|
|
171
205
|
_check_fraction(label, "seasonality", note["seasonality"])
|
|
172
206
|
|
|
207
|
+
if "distribution" in note:
|
|
208
|
+
if not is_numeric:
|
|
209
|
+
raise ValueError(f'{label}: "distribution" only applies to numeric columns.')
|
|
210
|
+
if note["distribution"] not in _DISTRIBUTIONS:
|
|
211
|
+
allowed = ", ".join(f'"{d}"' for d in _DISTRIBUTIONS)
|
|
212
|
+
raise ValueError(
|
|
213
|
+
f'{label}: "distribution" must be one of {allowed} (got {note["distribution"]!r}).'
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
# mean/stddev/median/spread only make sense alongside the distribution
|
|
217
|
+
# they shape, so each is checked against the note's own `distribution`
|
|
218
|
+
# (absent means "uniform", which none of them apply to either).
|
|
219
|
+
for param_key, required_distributions in _PARAM_DISTRIBUTIONS.items():
|
|
220
|
+
if param_key not in note:
|
|
221
|
+
continue
|
|
222
|
+
if not is_numeric:
|
|
223
|
+
raise ValueError(f'{label}: "{param_key}" only applies to numeric columns.')
|
|
224
|
+
if note.get("distribution") not in required_distributions:
|
|
225
|
+
options = " or ".join(f'"{d}"' for d in required_distributions)
|
|
226
|
+
raise ValueError(f'{label}: "{param_key}" only applies with "distribution": {options}.')
|
|
227
|
+
if param_key == "mean":
|
|
228
|
+
_check_number(label, param_key, note[param_key])
|
|
229
|
+
else:
|
|
230
|
+
_check_positive_number(label, param_key, note[param_key])
|
|
231
|
+
|
|
173
232
|
|
|
174
233
|
def _check_fraction(label: str, key: str, value: object) -> None:
|
|
175
234
|
if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0.0 <= value <= 1.0:
|
|
@@ -181,6 +240,16 @@ def _check_positive_int(label: str, key: str, value: object) -> None:
|
|
|
181
240
|
raise ValueError(f'{label}: "{key}" must be a positive whole number (got {value!r}).')
|
|
182
241
|
|
|
183
242
|
|
|
243
|
+
def _check_number(label: str, key: str, value: object) -> None:
|
|
244
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
245
|
+
raise ValueError(f'{label}: "{key}" must be a number (got {value!r}).')
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _check_positive_number(label: str, key: str, value: object) -> None:
|
|
249
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
|
|
250
|
+
raise ValueError(f'{label}: "{key}" must be a number greater than 0 (got {value!r}).')
|
|
251
|
+
|
|
252
|
+
|
|
184
253
|
def _check_bool(label: str, key: str, value: object) -> None:
|
|
185
254
|
if not isinstance(value, bool):
|
|
186
255
|
raise ValueError(f'{label}: "{key}" must be true or false (got {value!r}).')
|
|
@@ -36,8 +36,10 @@ tests/test_dbt_integration.py
|
|
|
36
36
|
tests/test_dbt_naming.py
|
|
37
37
|
tests/test_dbt_project.py
|
|
38
38
|
tests/test_dbt_tests.py
|
|
39
|
+
tests/test_distributions.py
|
|
39
40
|
tests/test_faker_name_inference.py
|
|
40
41
|
tests/test_generation.py
|
|
42
|
+
tests/test_lone_country.py
|
|
41
43
|
tests/test_options.py
|
|
42
44
|
tests/test_release_stress.py
|
|
43
45
|
tests/test_row_identity.py
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""Numeric distribution hints: `distribution` and its `mean`/`stddev`/`median`/`spread`
|
|
2
|
+
parameters on integer and decimal columns.
|
|
3
|
+
|
|
4
|
+
The product ask this closes was "can I make a normal distribution or another kind
|
|
5
|
+
of distribution as well?" on a numeric column -- `min`/`max` alone could only shape
|
|
6
|
+
a uniform spread. Every statistical test here uses a fixed seed and a generous
|
|
7
|
+
tolerance so it never flakes; the exact bounds were chosen by running the real
|
|
8
|
+
implementation and leaving headroom, not derived analytically.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import statistics
|
|
12
|
+
|
|
13
|
+
import pytest
|
|
14
|
+
|
|
15
|
+
from model2data.generate.core import generate_data_from_dbml
|
|
16
|
+
from model2data.parse.dbml import ColumnDef, TableDef
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _single_column_table(column: ColumnDef, table_name: str = "t") -> dict[str, TableDef]:
|
|
20
|
+
return {table_name: TableDef(name=table_name, columns=[ColumnDef("id", "int", {"pk"}), column])}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# ---------------------------------------------------------
|
|
24
|
+
# Step 0: pin today's (uniform) behaviour before touching it
|
|
25
|
+
# ---------------------------------------------------------
|
|
26
|
+
def test_no_distribution_hint_reproduces_pre_1_7_frames():
|
|
27
|
+
"""A numeric column with no `distribution` key must still produce exactly
|
|
28
|
+
what 1.6.0 produced with the same seed -- the guard rail every distribution
|
|
29
|
+
added here works against."""
|
|
30
|
+
tables = {
|
|
31
|
+
"t": TableDef(
|
|
32
|
+
name="t",
|
|
33
|
+
columns=[
|
|
34
|
+
ColumnDef("id", "int", {"pk"}),
|
|
35
|
+
ColumnDef("amount", "numeric", {"not null"}),
|
|
36
|
+
ColumnDef("qty", "int", {"not null"}),
|
|
37
|
+
],
|
|
38
|
+
)
|
|
39
|
+
}
|
|
40
|
+
data = generate_data_from_dbml(tables, [], base_rows=8, seed=777)
|
|
41
|
+
assert data["t"]["amount"].tolist() == [
|
|
42
|
+
4169.09,
|
|
43
|
+
3848.52,
|
|
44
|
+
5163.13,
|
|
45
|
+
3006.66,
|
|
46
|
+
25.14,
|
|
47
|
+
4363.61,
|
|
48
|
+
685.34,
|
|
49
|
+
8357.29,
|
|
50
|
+
]
|
|
51
|
+
assert data["t"]["qty"].tolist() == [7, 89, 51, 52, 82, 100, 97, 9]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# ---------------------------------------------------------
|
|
55
|
+
# Shape: each distribution draws roughly what it says it does
|
|
56
|
+
# ---------------------------------------------------------
|
|
57
|
+
def test_normal_distribution_matches_mean_and_stddev():
|
|
58
|
+
tables = _single_column_table(
|
|
59
|
+
ColumnDef(
|
|
60
|
+
"amount",
|
|
61
|
+
"numeric",
|
|
62
|
+
{"not null"},
|
|
63
|
+
note={"distribution": "normal", "mean": 100, "stddev": 10},
|
|
64
|
+
)
|
|
65
|
+
)
|
|
66
|
+
data = generate_data_from_dbml(tables, [], base_rows=5000, seed=1)
|
|
67
|
+
values = data["t"]["amount"].tolist()
|
|
68
|
+
assert 97 <= statistics.mean(values) <= 103
|
|
69
|
+
assert 8 <= statistics.stdev(values) <= 12
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_lognormal_distribution_median_and_mean():
|
|
73
|
+
tables = _single_column_table(
|
|
74
|
+
ColumnDef(
|
|
75
|
+
"amount",
|
|
76
|
+
"numeric",
|
|
77
|
+
{"not null"},
|
|
78
|
+
note={"distribution": "lognormal", "median": 80, "spread": 0.5},
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
data = generate_data_from_dbml(tables, [], base_rows=5000, seed=2)
|
|
82
|
+
values = data["t"]["amount"].tolist()
|
|
83
|
+
sample_median = statistics.median(values)
|
|
84
|
+
assert 70 <= sample_median <= 90
|
|
85
|
+
assert statistics.mean(values) > sample_median
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_exponential_distribution_matches_mean():
|
|
89
|
+
tables = _single_column_table(
|
|
90
|
+
ColumnDef(
|
|
91
|
+
"amount", "numeric", {"not null"}, note={"distribution": "exponential", "mean": 30}
|
|
92
|
+
)
|
|
93
|
+
)
|
|
94
|
+
data = generate_data_from_dbml(tables, [], base_rows=5000, seed=3)
|
|
95
|
+
values = data["t"]["amount"].tolist()
|
|
96
|
+
assert 26 <= statistics.mean(values) <= 34
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_missing_parameters_default_to_midpoint_and_derived_spread():
|
|
100
|
+
"""No `mean`/`stddev` at all: normal centres on the midpoint of the
|
|
101
|
+
default int range (0-100 -> 50) with stddev (max-min)/6."""
|
|
102
|
+
tables = _single_column_table(
|
|
103
|
+
ColumnDef("qty", "int", {"not null"}, note={"distribution": "normal"})
|
|
104
|
+
)
|
|
105
|
+
data = generate_data_from_dbml(tables, [], base_rows=5000, seed=7)
|
|
106
|
+
values = data["t"]["qty"].tolist()
|
|
107
|
+
assert 45 <= statistics.mean(values) <= 55
|
|
108
|
+
assert 12 <= statistics.stdev(values) <= 21
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_missing_lognormal_parameters_default_to_midpoint_and_half_spread():
|
|
112
|
+
"""No `median`/`spread`: lognormal centres on the midpoint of the default
|
|
113
|
+
decimal range (0-10,000 -> 5,000)."""
|
|
114
|
+
tables = _single_column_table(
|
|
115
|
+
ColumnDef("amount", "numeric", {"not null"}, note={"distribution": "lognormal"})
|
|
116
|
+
)
|
|
117
|
+
data = generate_data_from_dbml(tables, [], base_rows=5000, seed=8)
|
|
118
|
+
values = data["t"]["amount"].tolist()
|
|
119
|
+
assert 4000 <= statistics.median(values) <= 6000
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def test_missing_exponential_mean_defaults_to_midpoint():
|
|
123
|
+
"""No `mean`: exponential's scale defaults to the midpoint of the default
|
|
124
|
+
int range (0-100 -> 50)."""
|
|
125
|
+
tables = _single_column_table(
|
|
126
|
+
ColumnDef("qty", "int", {"not null"}, note={"distribution": "exponential"})
|
|
127
|
+
)
|
|
128
|
+
data = generate_data_from_dbml(tables, [], base_rows=5000, seed=9)
|
|
129
|
+
values = data["t"]["qty"].tolist()
|
|
130
|
+
assert 40 <= statistics.mean(values) <= 60
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
# ---------------------------------------------------------
|
|
134
|
+
# Clipping and typing
|
|
135
|
+
# ---------------------------------------------------------
|
|
136
|
+
def test_clipping_with_explicit_min_leaves_no_negatives():
|
|
137
|
+
"""A normal whose mean sits close to an explicit `min` would otherwise
|
|
138
|
+
draw negatives; redraw-then-clamp keeps every value in bounds."""
|
|
139
|
+
tables = _single_column_table(
|
|
140
|
+
ColumnDef(
|
|
141
|
+
"amount",
|
|
142
|
+
"numeric",
|
|
143
|
+
{"not null"},
|
|
144
|
+
note={"distribution": "normal", "mean": 5, "stddev": 10, "min": 0},
|
|
145
|
+
)
|
|
146
|
+
)
|
|
147
|
+
data = generate_data_from_dbml(tables, [], base_rows=2000, seed=4)
|
|
148
|
+
values = data["t"]["amount"].tolist()
|
|
149
|
+
assert min(values) >= 0
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def test_integer_columns_stay_ints_under_a_distribution():
|
|
153
|
+
tables = _single_column_table(
|
|
154
|
+
ColumnDef(
|
|
155
|
+
"qty", "int", {"not null"}, note={"distribution": "normal", "mean": 50, "stddev": 10}
|
|
156
|
+
)
|
|
157
|
+
)
|
|
158
|
+
data = generate_data_from_dbml(tables, [], base_rows=200, seed=5)
|
|
159
|
+
values = data["t"]["qty"].tolist()
|
|
160
|
+
assert all(isinstance(v, int) for v in values)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def test_unique_normal_column_stays_unique():
|
|
164
|
+
"""`ensure_unique` still resolves collisions when the generator draws
|
|
165
|
+
from a distribution instead of `random.sample` -- given a spread wide
|
|
166
|
+
enough to hold that many distinct values."""
|
|
167
|
+
tables = {
|
|
168
|
+
"t": TableDef(
|
|
169
|
+
name="t",
|
|
170
|
+
columns=[
|
|
171
|
+
ColumnDef(
|
|
172
|
+
"id",
|
|
173
|
+
"int",
|
|
174
|
+
{"pk"},
|
|
175
|
+
note={"distribution": "normal", "mean": 5000, "stddev": 500},
|
|
176
|
+
)
|
|
177
|
+
],
|
|
178
|
+
)
|
|
179
|
+
}
|
|
180
|
+
data = generate_data_from_dbml(tables, [], base_rows=200, seed=6)
|
|
181
|
+
values = data["t"]["id"].tolist()
|
|
182
|
+
assert len(set(values)) == len(values)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# ---------------------------------------------------------
|
|
186
|
+
# Validation: every error names the column
|
|
187
|
+
# ---------------------------------------------------------
|
|
188
|
+
def test_distribution_on_a_non_numeric_column_is_rejected():
|
|
189
|
+
tables = _single_column_table(ColumnDef("label", "varchar", note={"distribution": "normal"}))
|
|
190
|
+
with pytest.raises(ValueError, match=r"t\.label"):
|
|
191
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def test_distribution_rejects_an_unknown_value():
|
|
195
|
+
tables = _single_column_table(ColumnDef("total", "numeric", note={"distribution": "gaussian"}))
|
|
196
|
+
with pytest.raises(ValueError, match=r"t\.total"):
|
|
197
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def test_stddev_without_distribution_is_rejected():
|
|
201
|
+
tables = _single_column_table(ColumnDef("total", "numeric", note={"stddev": 5}))
|
|
202
|
+
with pytest.raises(
|
|
203
|
+
ValueError, match=r't\.total: "stddev" only applies with "distribution": "normal"\.'
|
|
204
|
+
):
|
|
205
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def test_stddev_with_the_wrong_distribution_is_rejected():
|
|
209
|
+
tables = _single_column_table(
|
|
210
|
+
ColumnDef("total", "numeric", note={"distribution": "exponential", "stddev": 5})
|
|
211
|
+
)
|
|
212
|
+
with pytest.raises(ValueError, match=r"t\.total"):
|
|
213
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def test_median_with_the_wrong_distribution_is_rejected():
|
|
217
|
+
tables = _single_column_table(
|
|
218
|
+
ColumnDef("total", "numeric", note={"distribution": "normal", "median": 5})
|
|
219
|
+
)
|
|
220
|
+
with pytest.raises(ValueError, match=r"t\.total"):
|
|
221
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def test_spread_with_the_wrong_distribution_is_rejected():
|
|
225
|
+
tables = _single_column_table(
|
|
226
|
+
ColumnDef("total", "numeric", note={"distribution": "normal", "spread": 0.5})
|
|
227
|
+
)
|
|
228
|
+
with pytest.raises(ValueError, match=r"t\.total"):
|
|
229
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def test_mean_without_a_matching_distribution_is_rejected():
|
|
233
|
+
tables = _single_column_table(ColumnDef("total", "numeric", note={"mean": 5}))
|
|
234
|
+
with pytest.raises(ValueError, match=r"t\.total"):
|
|
235
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
@pytest.mark.parametrize(
|
|
239
|
+
"note",
|
|
240
|
+
[
|
|
241
|
+
{"distribution": "normal", "stddev": -1},
|
|
242
|
+
{"distribution": "normal", "stddev": 0},
|
|
243
|
+
{"distribution": "lognormal", "median": -5},
|
|
244
|
+
{"distribution": "lognormal", "spread": 0},
|
|
245
|
+
],
|
|
246
|
+
)
|
|
247
|
+
def test_non_positive_stddev_median_spread_is_rejected(note):
|
|
248
|
+
tables = _single_column_table(ColumnDef("total", "numeric", note=note))
|
|
249
|
+
with pytest.raises(ValueError, match=r"t\.total"):
|
|
250
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def test_a_bool_value_is_rejected_where_a_number_is_expected():
|
|
254
|
+
tables = _single_column_table(
|
|
255
|
+
ColumnDef("total", "numeric", note={"distribution": "normal", "mean": True})
|
|
256
|
+
)
|
|
257
|
+
with pytest.raises(ValueError, match=r"t\.total"):
|
|
258
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""A lone `country` column reads as an international mix, not one repeated
|
|
2
|
+
country.
|
|
3
|
+
|
|
4
|
+
Since 1.3.0 every row draws one address from a per-table pool, and that
|
|
5
|
+
pool's country is always the locale's own -- right when a sibling
|
|
6
|
+
city/street/state/postcode column needs it to agree, wrong when `country` is
|
|
7
|
+
the only address-shaped column in the table. These tests pin: the mix (a
|
|
8
|
+
majority-but-not-all home-country share, several distinct countries), that a
|
|
9
|
+
sibling place column switches it back off, that a declared `country` type
|
|
10
|
+
behaves like the name, and that `distinct`/`null_rate`/determinism/locale
|
|
11
|
+
keep working the same way they do everywhere else.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from model2data.generate.core import generate_data_from_dbml
|
|
15
|
+
from model2data.parse.dbml import ColumnDef, TableDef
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _customers_table(*extra_columns: ColumnDef) -> TableDef:
|
|
19
|
+
return TableDef(
|
|
20
|
+
name="customers",
|
|
21
|
+
columns=[
|
|
22
|
+
ColumnDef(name="id", data_type="int", settings={"pk"}),
|
|
23
|
+
ColumnDef(name="country", data_type="varchar", settings={"not null"}),
|
|
24
|
+
*extra_columns,
|
|
25
|
+
],
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class TestLoneCountryColumn:
|
|
30
|
+
def test_lone_country_column_reads_as_an_international_mix(self):
|
|
31
|
+
df = generate_data_from_dbml(
|
|
32
|
+
tables={"customers": _customers_table()},
|
|
33
|
+
refs=[],
|
|
34
|
+
base_rows=500,
|
|
35
|
+
seed=1,
|
|
36
|
+
locale="nl_BE",
|
|
37
|
+
)["customers"]
|
|
38
|
+
|
|
39
|
+
counts = df["country"].value_counts(normalize=True)
|
|
40
|
+
assert len(counts) >= 5
|
|
41
|
+
|
|
42
|
+
home_share = counts.get("Belgium", 0.0)
|
|
43
|
+
assert 0.5 <= home_share <= 0.7
|
|
44
|
+
assert counts.idxmax() == "Belgium"
|
|
45
|
+
|
|
46
|
+
def test_a_sibling_place_column_keeps_the_country_single(self):
|
|
47
|
+
df = generate_data_from_dbml(
|
|
48
|
+
tables={
|
|
49
|
+
"customers": _customers_table(
|
|
50
|
+
ColumnDef(name="city", data_type="varchar", settings={"not null"})
|
|
51
|
+
)
|
|
52
|
+
},
|
|
53
|
+
refs=[],
|
|
54
|
+
base_rows=200,
|
|
55
|
+
seed=2,
|
|
56
|
+
locale="nl_BE",
|
|
57
|
+
)["customers"]
|
|
58
|
+
|
|
59
|
+
assert set(df["country"]) == {"Belgium"}
|
|
60
|
+
|
|
61
|
+
def test_declared_country_type_behaves_like_the_name(self):
|
|
62
|
+
df = generate_data_from_dbml(
|
|
63
|
+
tables={
|
|
64
|
+
"customers": TableDef(
|
|
65
|
+
name="customers",
|
|
66
|
+
columns=[
|
|
67
|
+
ColumnDef(name="id", data_type="int", settings={"pk"}),
|
|
68
|
+
# Named generically; the *declared type* is what
|
|
69
|
+
# says "country".
|
|
70
|
+
ColumnDef(name="hq", data_type="country", settings={"not null"}),
|
|
71
|
+
],
|
|
72
|
+
)
|
|
73
|
+
},
|
|
74
|
+
refs=[],
|
|
75
|
+
base_rows=500,
|
|
76
|
+
seed=3,
|
|
77
|
+
locale="nl_BE",
|
|
78
|
+
)["customers"]
|
|
79
|
+
|
|
80
|
+
counts = df["hq"].value_counts(normalize=True)
|
|
81
|
+
assert len(counts) >= 5
|
|
82
|
+
assert 0.5 <= counts.get("Belgium", 0.0) <= 0.7
|
|
83
|
+
|
|
84
|
+
def test_distinct_hint_still_bounds_the_pool(self):
|
|
85
|
+
df = generate_data_from_dbml(
|
|
86
|
+
tables={
|
|
87
|
+
"customers": TableDef(
|
|
88
|
+
name="customers",
|
|
89
|
+
columns=[
|
|
90
|
+
ColumnDef(name="id", data_type="int", settings={"pk"}),
|
|
91
|
+
ColumnDef(
|
|
92
|
+
name="country",
|
|
93
|
+
data_type="varchar",
|
|
94
|
+
settings={"not null"},
|
|
95
|
+
note={"distinct": 3},
|
|
96
|
+
),
|
|
97
|
+
],
|
|
98
|
+
)
|
|
99
|
+
},
|
|
100
|
+
refs=[],
|
|
101
|
+
base_rows=300,
|
|
102
|
+
seed=4,
|
|
103
|
+
locale="nl_BE",
|
|
104
|
+
)["customers"]
|
|
105
|
+
|
|
106
|
+
assert df["country"].nunique() <= 3
|
|
107
|
+
|
|
108
|
+
def test_same_seed_reproduces_the_same_mix(self):
|
|
109
|
+
def run():
|
|
110
|
+
return generate_data_from_dbml(
|
|
111
|
+
tables={"customers": _customers_table()},
|
|
112
|
+
refs=[],
|
|
113
|
+
base_rows=200,
|
|
114
|
+
seed=5,
|
|
115
|
+
locale="nl_BE",
|
|
116
|
+
)["customers"]
|
|
117
|
+
|
|
118
|
+
first_run, second_run = run(), run()
|
|
119
|
+
assert list(first_run["country"]) == list(second_run["country"])
|
|
120
|
+
|
|
121
|
+
def test_default_locale_also_produces_a_mix(self):
|
|
122
|
+
df = generate_data_from_dbml(
|
|
123
|
+
tables={"customers": _customers_table()},
|
|
124
|
+
refs=[],
|
|
125
|
+
base_rows=500,
|
|
126
|
+
seed=6,
|
|
127
|
+
)["customers"]
|
|
128
|
+
|
|
129
|
+
counts = df["country"].value_counts(normalize=True)
|
|
130
|
+
assert len(counts) >= 5
|
|
131
|
+
assert 0.5 <= counts.get("United States", 0.0) <= 0.7
|
|
132
|
+
assert counts.idxmax() == "United States"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{model2data-1.6.0 → model2data-1.7.1}/model2data/dbt/templates/macros/generate_schema_name.sql
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|