model2data 1.6.0__tar.gz → 1.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {model2data-1.6.0/model2data.egg-info → model2data-1.7.0}/PKG-INFO +1 -1
- {model2data-1.6.0 → model2data-1.7.0}/README.md +20 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/generate/faker.py +116 -8
- {model2data-1.6.0 → model2data-1.7.0}/model2data/generate/hints.py +73 -4
- {model2data-1.6.0 → model2data-1.7.0/model2data.egg-info}/PKG-INFO +1 -1
- {model2data-1.6.0 → model2data-1.7.0}/model2data.egg-info/SOURCES.txt +1 -0
- {model2data-1.6.0 → model2data-1.7.0}/pyproject.toml +1 -1
- model2data-1.7.0/tests/test_distributions.py +258 -0
- {model2data-1.6.0 → model2data-1.7.0}/LICENSE +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/README_PYPI.md +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/__init__.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/cli.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/dbt/__init__.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/dbt/project.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/dbt/templates/dbt_project.yml.jinja +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/dbt/templates/macros/generate_schema_name.sql +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/dbt/templates/profiles.yml.jinja +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/dbt/tests.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/generate/__init__.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/generate/core.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/generate/options.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/generate/relationships.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/generate/timeline.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/parse/__init__.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/parse/dbml.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data/utils.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data.egg-info/dependency_links.txt +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data.egg-info/entry_points.txt +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data.egg-info/requires.txt +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/model2data.egg-info/top_level.txt +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/setup.cfg +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_as_of_anchor.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_cli.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_column_time_hints.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_coverage_gaps.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_dbml_parser.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_dbml_parser_fuzz.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_dbt_integration.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_dbt_naming.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_dbt_project.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_dbt_tests.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_faker_name_inference.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_generation.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_options.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_release_stress.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_row_identity.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_shaping.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_table_seeds.py +0 -0
- {model2data-1.6.0 → model2data-1.7.0}/tests/test_timeline.py +0 -0
|
@@ -232,6 +232,26 @@ non-null rows a boolean column comes back `true`. `null_rate` replaces the colum
|
|
|
232
232
|
fraction outright. `distinct` draws the column's values from a fixed-size pool instead of a fresh
|
|
233
233
|
value per row — a `shipping_city` most warehouses only ever see a handful of.
|
|
234
234
|
|
|
235
|
+
### Shape a number's distribution
|
|
236
|
+
|
|
237
|
+
`min`/`max` alone only ever drew uniformly between them. A `distribution` note hint on an
|
|
238
|
+
integer or decimal column picks a different shape instead:
|
|
239
|
+
|
|
240
|
+
```dbml
|
|
241
|
+
Table orders {
|
|
242
|
+
id int [pk]
|
|
243
|
+
total_amount numeric [note: '{"distribution": "lognormal", "median": 80, "spread": 0.6, "min": 5}']
|
|
244
|
+
}
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
`normal` takes `mean` (the centre) and `stddev` (the spread); `lognormal` takes `median` (the
|
|
248
|
+
typical value) and `spread` (how heavy the tail is — 0.3 is mild, 1.0 is heavy); `exponential`
|
|
249
|
+
takes `mean` (the average). Any left unset default to the midpoint of the column's effective
|
|
250
|
+
`min`/`max` (or `stddev` = range / 6, `spread` = 0.5). `min`/`max` still clip the result — a
|
|
251
|
+
`normal` centred near an edge redraws a bounded number of times before clamping, so it never
|
|
252
|
+
loops forever and never crosses the bound. Leaving `distribution` out, or setting it to
|
|
253
|
+
`"uniform"`, is exactly today's behaviour.
|
|
254
|
+
|
|
235
255
|
Run dbt to load, transform, and test the data:
|
|
236
256
|
|
|
237
257
|
```bash
|
|
@@ -665,6 +665,13 @@ def generate_column_values(
|
|
|
665
665
|
if column.note:
|
|
666
666
|
min_val = column.note.get("min")
|
|
667
667
|
max_val = column.note.get("max")
|
|
668
|
+
# The note's own min/max, before either numeric branch fills in its
|
|
669
|
+
# branch-specific default -- these are the only bounds a distribution
|
|
670
|
+
# hint clips against (see `_clipped_distribution_draw`). A column with no
|
|
671
|
+
# explicit min/max draws from an unclipped distribution even though the
|
|
672
|
+
# branch below still has an implicit default range for the *parameters*.
|
|
673
|
+
explicit_min, explicit_max = min_val, max_val
|
|
674
|
+
distribution = column_note.get("distribution", "uniform")
|
|
668
675
|
|
|
669
676
|
if fk_series is not None and not fk_series.empty:
|
|
670
677
|
# A plain branch of the same if/elif chain (rather than an early
|
|
@@ -722,7 +729,22 @@ def generate_column_values(
|
|
|
722
729
|
if max_val is None:
|
|
723
730
|
max_val = 100
|
|
724
731
|
|
|
725
|
-
if
|
|
732
|
+
if distribution != "uniform":
|
|
733
|
+
# A shaped column never needs the row_count-sized value space
|
|
734
|
+
# random.sample relies on below, so it always draws one value at
|
|
735
|
+
# a time -- unique or not -- and leans on _deduplicate the same
|
|
736
|
+
# way every other non-uniform-fast-path branch does.
|
|
737
|
+
params = _distribution_params(distribution, min_val, max_val, column_note)
|
|
738
|
+
|
|
739
|
+
def generator() -> int:
|
|
740
|
+
return round(
|
|
741
|
+
_clipped_distribution_draw(distribution, params, explicit_min, explicit_max)
|
|
742
|
+
)
|
|
743
|
+
|
|
744
|
+
values = [generator() for _ in range(row_count)]
|
|
745
|
+
if ensure_unique:
|
|
746
|
+
values = _deduplicate(values, generator, column_name=unique_label)
|
|
747
|
+
elif ensure_unique:
|
|
726
748
|
if not had_explicit_range:
|
|
727
749
|
# No user-specified range: widen the default so there's
|
|
728
750
|
# always enough headroom for `row_count` unique PK values.
|
|
@@ -753,13 +775,27 @@ def generate_column_values(
|
|
|
753
775
|
min_val = 0
|
|
754
776
|
if max_val is None:
|
|
755
777
|
max_val = 10_000
|
|
756
|
-
|
|
757
|
-
if
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
778
|
+
|
|
779
|
+
if distribution != "uniform":
|
|
780
|
+
params = _distribution_params(distribution, min_val, max_val, column_note)
|
|
781
|
+
|
|
782
|
+
def generator() -> float:
|
|
783
|
+
return round(
|
|
784
|
+
_clipped_distribution_draw(distribution, params, explicit_min, explicit_max),
|
|
785
|
+
2,
|
|
786
|
+
)
|
|
787
|
+
|
|
788
|
+
values = [generator() for _ in range(row_count)]
|
|
789
|
+
if ensure_unique:
|
|
790
|
+
values = _deduplicate(values, generator, column_name=unique_label)
|
|
791
|
+
else:
|
|
792
|
+
values = [round(random.uniform(min_val, max_val), 2) for _ in range(row_count)]
|
|
793
|
+
if ensure_unique:
|
|
794
|
+
values = _deduplicate(
|
|
795
|
+
values,
|
|
796
|
+
lambda: round(random.uniform(min_val, max_val), 2),
|
|
797
|
+
column_name=unique_label,
|
|
798
|
+
)
|
|
763
799
|
|
|
764
800
|
# -----------------------------------------------------
|
|
765
801
|
# Booleans
|
|
@@ -824,6 +860,78 @@ def generate_column_values(
|
|
|
824
860
|
# ---------------------------------------------------------
|
|
825
861
|
# Internal helpers
|
|
826
862
|
# ---------------------------------------------------------
|
|
863
|
+
# How many times a clipped draw redraws before giving up and clamping. Kept
|
|
864
|
+
# small and fixed rather than exposed as a hint: a normal whose mean sits
|
|
865
|
+
# well inside its min/max rarely needs a redraw at all, and a distribution
|
|
866
|
+
# whose parameters put most of its mass outside the bounds is a schema
|
|
867
|
+
# mistake no retry count fixes -- clamping is the honest fallback either way.
|
|
868
|
+
_MAX_DISTRIBUTION_REDRAWS = 20
|
|
869
|
+
|
|
870
|
+
|
|
871
|
+
def _distribution_params(
|
|
872
|
+
distribution: str, effective_min: float, effective_max: float, note: dict
|
|
873
|
+
) -> dict:
|
|
874
|
+
"""Fill in whichever of mean/stddev/median/spread `note` leaves unset.
|
|
875
|
+
|
|
876
|
+
`effective_min`/`effective_max` are the note's own `min`/`max` when given,
|
|
877
|
+
else the calling branch's default range (0-100 for integers, 0-10,000 for
|
|
878
|
+
decimals) -- ordinary enough a midpoint to centre an unparameterized
|
|
879
|
+
distribution on.
|
|
880
|
+
"""
|
|
881
|
+
midpoint = (effective_min + effective_max) / 2
|
|
882
|
+
if distribution == "normal":
|
|
883
|
+
return {
|
|
884
|
+
"mean": note.get("mean", midpoint),
|
|
885
|
+
"stddev": note.get("stddev", (effective_max - effective_min) / 6),
|
|
886
|
+
}
|
|
887
|
+
if distribution == "lognormal":
|
|
888
|
+
return {
|
|
889
|
+
"median": note.get("median", midpoint),
|
|
890
|
+
"spread": note.get("spread", 0.5),
|
|
891
|
+
}
|
|
892
|
+
# exponential: hints.py has already rejected anything else reaching here.
|
|
893
|
+
return {"mean": note.get("mean", midpoint)}
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
def _draw_distribution_value(distribution: str, params: dict) -> float:
|
|
897
|
+
"""One raw draw from `distribution`, unclipped and unrounded."""
|
|
898
|
+
if distribution == "normal":
|
|
899
|
+
return random.gauss(params["mean"], params["stddev"])
|
|
900
|
+
if distribution == "lognormal":
|
|
901
|
+
return random.lognormvariate(math.log(params["median"]), params["spread"])
|
|
902
|
+
return random.expovariate(1 / params["mean"]) # exponential
|
|
903
|
+
|
|
904
|
+
|
|
905
|
+
def _clipped_distribution_draw(
|
|
906
|
+
distribution: str,
|
|
907
|
+
params: dict,
|
|
908
|
+
clip_min: Optional[float],
|
|
909
|
+
clip_max: Optional[float],
|
|
910
|
+
) -> float:
|
|
911
|
+
"""Draw from `distribution`, redrawing out-of-bounds values before clamping.
|
|
912
|
+
|
|
913
|
+
A normal centred well inside [min, max] almost never needs the clamp; one
|
|
914
|
+
centred near an edge (mean 120, min 0) would otherwise pile values up at
|
|
915
|
+
the boundary, so a bounded number of redraws is tried first and only a
|
|
916
|
+
value still out of bounds after all of them gets clamped -- which also
|
|
917
|
+
keeps this from looping forever when min and max leave no room at all.
|
|
918
|
+
`clip_min`/`clip_max` are the note's own bounds (`None` when the column
|
|
919
|
+
didn't set one), never the branch's implicit default range.
|
|
920
|
+
"""
|
|
921
|
+
value = _draw_distribution_value(distribution, params)
|
|
922
|
+
attempts = 0
|
|
923
|
+
while attempts < _MAX_DISTRIBUTION_REDRAWS and (
|
|
924
|
+
(clip_min is not None and value < clip_min) or (clip_max is not None and value > clip_max)
|
|
925
|
+
):
|
|
926
|
+
value = _draw_distribution_value(distribution, params)
|
|
927
|
+
attempts += 1
|
|
928
|
+
if clip_min is not None and value < clip_min:
|
|
929
|
+
value = clip_min
|
|
930
|
+
if clip_max is not None and value > clip_max:
|
|
931
|
+
value = clip_max
|
|
932
|
+
return value
|
|
933
|
+
|
|
934
|
+
|
|
827
935
|
def _null_fraction_for(column: ColumnDef, row_count: int) -> float:
|
|
828
936
|
"""The fraction of `row_count` rows this column should turn null.
|
|
829
937
|
|
|
@@ -3,10 +3,11 @@
|
|
|
3
3
|
A note has always been either plain text (a comment, ignored by generation)
|
|
4
4
|
or a JSON object read for `min`/`max`. This module documents the rest of that
|
|
5
5
|
object's vocabulary -- `null_rate`, `weights`, `true_rate`, `distinct`, `skew`,
|
|
6
|
-
`after`, `business_hours`, `growth`, `seasonality`
|
|
7
|
-
before a single row is generated, so a
|
|
8
|
-
wrong kind of column fails with a message
|
|
9
|
-
than surfacing as a wrong-looking dataset
|
|
6
|
+
`after`, `business_hours`, `growth`, `seasonality`, `distribution` and its
|
|
7
|
+
parameters -- and checks it once, before a single row is generated, so a
|
|
8
|
+
typo'd enum value or a hint on the wrong kind of column fails with a message
|
|
9
|
+
naming the table and column rather than surfacing as a wrong-looking dataset
|
|
10
|
+
or a downstream dbt test failure.
|
|
10
11
|
|
|
11
12
|
| key | applies to | meaning |
|
|
12
13
|
|-------------|-----------------------------------------------|--------------------------------------------|
|
|
@@ -20,6 +21,11 @@ than surfacing as a wrong-looking dataset or a downstream dbt test failure.
|
|
|
20
21
|
| `business_hours` | date/timestamp columns | overrides the run-level `TimeProfile.business_hours` for this column |
|
|
21
22
|
| `growth` | date/timestamp columns | overrides the run-level `TimeProfile.growth` for this column |
|
|
22
23
|
| `seasonality` | date/timestamp columns | overrides the run-level `TimeProfile.seasonality` for this column |
|
|
24
|
+
| `distribution` | numeric columns | draw shape: `uniform` (default), `normal`, `lognormal`, `exponential` |
|
|
25
|
+
| `mean` | numeric columns, with `distribution: normal` or `exponential` | normal: the centre; exponential: the average (scale) |
|
|
26
|
+
| `stddev` | numeric columns, with `distribution: normal` | normal: the spread |
|
|
27
|
+
| `median` | numeric columns, with `distribution: lognormal`| lognormal: the typical value, `exp(mu)` |
|
|
28
|
+
| `spread` | numeric columns, with `distribution: lognormal`| lognormal: sigma of the underlying normal (0.3 mild, 1.0 heavy tail) |
|
|
23
29
|
"""
|
|
24
30
|
|
|
25
31
|
from __future__ import annotations
|
|
@@ -95,6 +101,33 @@ def _is_temporal_type(base_type: str) -> bool:
|
|
|
95
101
|
return any(key in base_type for key in ("timestamp", "datetime"))
|
|
96
102
|
|
|
97
103
|
|
|
104
|
+
def _is_numeric_type(base_type: str) -> bool:
|
|
105
|
+
"""True for the integer and float/decimal types `min`/`max`/`distribution` apply to.
|
|
106
|
+
|
|
107
|
+
Mirrors generate.faker's own two numeric branches exactly, so a hint this
|
|
108
|
+
module accepts is guaranteed to land on a branch that reads it.
|
|
109
|
+
"""
|
|
110
|
+
return any(
|
|
111
|
+
key in base_type
|
|
112
|
+
for key in ("int", "integer", "bigint", "smallint", "decimal", "numeric", "float", "double")
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# Distributions a numeric column's `distribution` hint may name.
|
|
117
|
+
_DISTRIBUTIONS = ("uniform", "normal", "lognormal", "exponential")
|
|
118
|
+
|
|
119
|
+
# Which distribution(s) each shape parameter is meaningful under -- checked
|
|
120
|
+
# against the note's own `distribution` (default "uniform" when absent), so
|
|
121
|
+
# a `stddev` with no `distribution` key is rejected the same as one paired
|
|
122
|
+
# with the wrong distribution.
|
|
123
|
+
_PARAM_DISTRIBUTIONS: dict[str, tuple[str, ...]] = {
|
|
124
|
+
"mean": ("normal", "exponential"),
|
|
125
|
+
"stddev": ("normal",),
|
|
126
|
+
"median": ("lognormal",),
|
|
127
|
+
"spread": ("lognormal",),
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
|
|
98
131
|
def _validate_column_hints(
|
|
99
132
|
table_name: str,
|
|
100
133
|
column: ColumnDef,
|
|
@@ -110,6 +143,7 @@ def _validate_column_hints(
|
|
|
110
143
|
is_enum = bool(column.enum_values)
|
|
111
144
|
is_boolean = "boolean" in base_type or "bool" in base_type
|
|
112
145
|
is_temporal = _is_temporal_type(base_type)
|
|
146
|
+
is_numeric = _is_numeric_type(base_type)
|
|
113
147
|
is_nullable = not is_pk and "not null" not in column.settings
|
|
114
148
|
|
|
115
149
|
if "null_rate" in note:
|
|
@@ -170,6 +204,31 @@ def _validate_column_hints(
|
|
|
170
204
|
raise ValueError(f'{label}: "seasonality" only applies to date/timestamp columns.')
|
|
171
205
|
_check_fraction(label, "seasonality", note["seasonality"])
|
|
172
206
|
|
|
207
|
+
if "distribution" in note:
|
|
208
|
+
if not is_numeric:
|
|
209
|
+
raise ValueError(f'{label}: "distribution" only applies to numeric columns.')
|
|
210
|
+
if note["distribution"] not in _DISTRIBUTIONS:
|
|
211
|
+
allowed = ", ".join(f'"{d}"' for d in _DISTRIBUTIONS)
|
|
212
|
+
raise ValueError(
|
|
213
|
+
f'{label}: "distribution" must be one of {allowed} (got {note["distribution"]!r}).'
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
# mean/stddev/median/spread only make sense alongside the distribution
|
|
217
|
+
# they shape, so each is checked against the note's own `distribution`
|
|
218
|
+
# (absent means "uniform", which none of them apply to either).
|
|
219
|
+
for param_key, required_distributions in _PARAM_DISTRIBUTIONS.items():
|
|
220
|
+
if param_key not in note:
|
|
221
|
+
continue
|
|
222
|
+
if not is_numeric:
|
|
223
|
+
raise ValueError(f'{label}: "{param_key}" only applies to numeric columns.')
|
|
224
|
+
if note.get("distribution") not in required_distributions:
|
|
225
|
+
options = " or ".join(f'"{d}"' for d in required_distributions)
|
|
226
|
+
raise ValueError(f'{label}: "{param_key}" only applies with "distribution": {options}.')
|
|
227
|
+
if param_key == "mean":
|
|
228
|
+
_check_number(label, param_key, note[param_key])
|
|
229
|
+
else:
|
|
230
|
+
_check_positive_number(label, param_key, note[param_key])
|
|
231
|
+
|
|
173
232
|
|
|
174
233
|
def _check_fraction(label: str, key: str, value: object) -> None:
|
|
175
234
|
if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0.0 <= value <= 1.0:
|
|
@@ -181,6 +240,16 @@ def _check_positive_int(label: str, key: str, value: object) -> None:
|
|
|
181
240
|
raise ValueError(f'{label}: "{key}" must be a positive whole number (got {value!r}).')
|
|
182
241
|
|
|
183
242
|
|
|
243
|
+
def _check_number(label: str, key: str, value: object) -> None:
|
|
244
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
245
|
+
raise ValueError(f'{label}: "{key}" must be a number (got {value!r}).')
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _check_positive_number(label: str, key: str, value: object) -> None:
|
|
249
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
|
|
250
|
+
raise ValueError(f'{label}: "{key}" must be a number greater than 0 (got {value!r}).')
|
|
251
|
+
|
|
252
|
+
|
|
184
253
|
def _check_bool(label: str, key: str, value: object) -> None:
|
|
185
254
|
if not isinstance(value, bool):
|
|
186
255
|
raise ValueError(f'{label}: "{key}" must be true or false (got {value!r}).')
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""Numeric distribution hints: `distribution` and its `mean`/`stddev`/`median`/`spread`
|
|
2
|
+
parameters on integer and decimal columns.
|
|
3
|
+
|
|
4
|
+
The product ask this closes was "can I make a normal distribution or another kind
|
|
5
|
+
of distribution as well?" on a numeric column -- `min`/`max` alone could only shape
|
|
6
|
+
a uniform spread. Every statistical test here uses a fixed seed and a generous
|
|
7
|
+
tolerance so it never flakes; the exact bounds were chosen by running the real
|
|
8
|
+
implementation and leaving headroom, not derived analytically.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import statistics
|
|
12
|
+
|
|
13
|
+
import pytest
|
|
14
|
+
|
|
15
|
+
from model2data.generate.core import generate_data_from_dbml
|
|
16
|
+
from model2data.parse.dbml import ColumnDef, TableDef
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _single_column_table(column: ColumnDef, table_name: str = "t") -> dict[str, TableDef]:
|
|
20
|
+
return {table_name: TableDef(name=table_name, columns=[ColumnDef("id", "int", {"pk"}), column])}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# ---------------------------------------------------------
|
|
24
|
+
# Step 0: pin today's (uniform) behaviour before touching it
|
|
25
|
+
# ---------------------------------------------------------
|
|
26
|
+
def test_no_distribution_hint_reproduces_pre_1_7_frames():
|
|
27
|
+
"""A numeric column with no `distribution` key must still produce exactly
|
|
28
|
+
what 1.6.0 produced with the same seed -- the guard rail every distribution
|
|
29
|
+
added here works against."""
|
|
30
|
+
tables = {
|
|
31
|
+
"t": TableDef(
|
|
32
|
+
name="t",
|
|
33
|
+
columns=[
|
|
34
|
+
ColumnDef("id", "int", {"pk"}),
|
|
35
|
+
ColumnDef("amount", "numeric", {"not null"}),
|
|
36
|
+
ColumnDef("qty", "int", {"not null"}),
|
|
37
|
+
],
|
|
38
|
+
)
|
|
39
|
+
}
|
|
40
|
+
data = generate_data_from_dbml(tables, [], base_rows=8, seed=777)
|
|
41
|
+
assert data["t"]["amount"].tolist() == [
|
|
42
|
+
4169.09,
|
|
43
|
+
3848.52,
|
|
44
|
+
5163.13,
|
|
45
|
+
3006.66,
|
|
46
|
+
25.14,
|
|
47
|
+
4363.61,
|
|
48
|
+
685.34,
|
|
49
|
+
8357.29,
|
|
50
|
+
]
|
|
51
|
+
assert data["t"]["qty"].tolist() == [7, 89, 51, 52, 82, 100, 97, 9]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# ---------------------------------------------------------
|
|
55
|
+
# Shape: each distribution draws roughly what it says it does
|
|
56
|
+
# ---------------------------------------------------------
|
|
57
|
+
def test_normal_distribution_matches_mean_and_stddev():
|
|
58
|
+
tables = _single_column_table(
|
|
59
|
+
ColumnDef(
|
|
60
|
+
"amount",
|
|
61
|
+
"numeric",
|
|
62
|
+
{"not null"},
|
|
63
|
+
note={"distribution": "normal", "mean": 100, "stddev": 10},
|
|
64
|
+
)
|
|
65
|
+
)
|
|
66
|
+
data = generate_data_from_dbml(tables, [], base_rows=5000, seed=1)
|
|
67
|
+
values = data["t"]["amount"].tolist()
|
|
68
|
+
assert 97 <= statistics.mean(values) <= 103
|
|
69
|
+
assert 8 <= statistics.stdev(values) <= 12
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_lognormal_distribution_median_and_mean():
|
|
73
|
+
tables = _single_column_table(
|
|
74
|
+
ColumnDef(
|
|
75
|
+
"amount",
|
|
76
|
+
"numeric",
|
|
77
|
+
{"not null"},
|
|
78
|
+
note={"distribution": "lognormal", "median": 80, "spread": 0.5},
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
data = generate_data_from_dbml(tables, [], base_rows=5000, seed=2)
|
|
82
|
+
values = data["t"]["amount"].tolist()
|
|
83
|
+
sample_median = statistics.median(values)
|
|
84
|
+
assert 70 <= sample_median <= 90
|
|
85
|
+
assert statistics.mean(values) > sample_median
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_exponential_distribution_matches_mean():
|
|
89
|
+
tables = _single_column_table(
|
|
90
|
+
ColumnDef(
|
|
91
|
+
"amount", "numeric", {"not null"}, note={"distribution": "exponential", "mean": 30}
|
|
92
|
+
)
|
|
93
|
+
)
|
|
94
|
+
data = generate_data_from_dbml(tables, [], base_rows=5000, seed=3)
|
|
95
|
+
values = data["t"]["amount"].tolist()
|
|
96
|
+
assert 26 <= statistics.mean(values) <= 34
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_missing_parameters_default_to_midpoint_and_derived_spread():
|
|
100
|
+
"""No `mean`/`stddev` at all: normal centres on the midpoint of the
|
|
101
|
+
default int range (0-100 -> 50) with stddev (max-min)/6."""
|
|
102
|
+
tables = _single_column_table(
|
|
103
|
+
ColumnDef("qty", "int", {"not null"}, note={"distribution": "normal"})
|
|
104
|
+
)
|
|
105
|
+
data = generate_data_from_dbml(tables, [], base_rows=5000, seed=7)
|
|
106
|
+
values = data["t"]["qty"].tolist()
|
|
107
|
+
assert 45 <= statistics.mean(values) <= 55
|
|
108
|
+
assert 12 <= statistics.stdev(values) <= 21
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_missing_lognormal_parameters_default_to_midpoint_and_half_spread():
|
|
112
|
+
"""No `median`/`spread`: lognormal centres on the midpoint of the default
|
|
113
|
+
decimal range (0-10,000 -> 5,000)."""
|
|
114
|
+
tables = _single_column_table(
|
|
115
|
+
ColumnDef("amount", "numeric", {"not null"}, note={"distribution": "lognormal"})
|
|
116
|
+
)
|
|
117
|
+
data = generate_data_from_dbml(tables, [], base_rows=5000, seed=8)
|
|
118
|
+
values = data["t"]["amount"].tolist()
|
|
119
|
+
assert 4000 <= statistics.median(values) <= 6000
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def test_missing_exponential_mean_defaults_to_midpoint():
|
|
123
|
+
"""No `mean`: exponential's scale defaults to the midpoint of the default
|
|
124
|
+
int range (0-100 -> 50)."""
|
|
125
|
+
tables = _single_column_table(
|
|
126
|
+
ColumnDef("qty", "int", {"not null"}, note={"distribution": "exponential"})
|
|
127
|
+
)
|
|
128
|
+
data = generate_data_from_dbml(tables, [], base_rows=5000, seed=9)
|
|
129
|
+
values = data["t"]["qty"].tolist()
|
|
130
|
+
assert 40 <= statistics.mean(values) <= 60
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
# ---------------------------------------------------------
|
|
134
|
+
# Clipping and typing
|
|
135
|
+
# ---------------------------------------------------------
|
|
136
|
+
def test_clipping_with_explicit_min_leaves_no_negatives():
|
|
137
|
+
"""A normal whose mean sits close to an explicit `min` would otherwise
|
|
138
|
+
draw negatives; redraw-then-clamp keeps every value in bounds."""
|
|
139
|
+
tables = _single_column_table(
|
|
140
|
+
ColumnDef(
|
|
141
|
+
"amount",
|
|
142
|
+
"numeric",
|
|
143
|
+
{"not null"},
|
|
144
|
+
note={"distribution": "normal", "mean": 5, "stddev": 10, "min": 0},
|
|
145
|
+
)
|
|
146
|
+
)
|
|
147
|
+
data = generate_data_from_dbml(tables, [], base_rows=2000, seed=4)
|
|
148
|
+
values = data["t"]["amount"].tolist()
|
|
149
|
+
assert min(values) >= 0
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def test_integer_columns_stay_ints_under_a_distribution():
|
|
153
|
+
tables = _single_column_table(
|
|
154
|
+
ColumnDef(
|
|
155
|
+
"qty", "int", {"not null"}, note={"distribution": "normal", "mean": 50, "stddev": 10}
|
|
156
|
+
)
|
|
157
|
+
)
|
|
158
|
+
data = generate_data_from_dbml(tables, [], base_rows=200, seed=5)
|
|
159
|
+
values = data["t"]["qty"].tolist()
|
|
160
|
+
assert all(isinstance(v, int) for v in values)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def test_unique_normal_column_stays_unique():
|
|
164
|
+
"""`ensure_unique` still resolves collisions when the generator draws
|
|
165
|
+
from a distribution instead of `random.sample` -- given a spread wide
|
|
166
|
+
enough to hold that many distinct values."""
|
|
167
|
+
tables = {
|
|
168
|
+
"t": TableDef(
|
|
169
|
+
name="t",
|
|
170
|
+
columns=[
|
|
171
|
+
ColumnDef(
|
|
172
|
+
"id",
|
|
173
|
+
"int",
|
|
174
|
+
{"pk"},
|
|
175
|
+
note={"distribution": "normal", "mean": 5000, "stddev": 500},
|
|
176
|
+
)
|
|
177
|
+
],
|
|
178
|
+
)
|
|
179
|
+
}
|
|
180
|
+
data = generate_data_from_dbml(tables, [], base_rows=200, seed=6)
|
|
181
|
+
values = data["t"]["id"].tolist()
|
|
182
|
+
assert len(set(values)) == len(values)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# ---------------------------------------------------------
|
|
186
|
+
# Validation: every error names the column
|
|
187
|
+
# ---------------------------------------------------------
|
|
188
|
+
def test_distribution_on_a_non_numeric_column_is_rejected():
|
|
189
|
+
tables = _single_column_table(ColumnDef("label", "varchar", note={"distribution": "normal"}))
|
|
190
|
+
with pytest.raises(ValueError, match=r"t\.label"):
|
|
191
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def test_distribution_rejects_an_unknown_value():
|
|
195
|
+
tables = _single_column_table(ColumnDef("total", "numeric", note={"distribution": "gaussian"}))
|
|
196
|
+
with pytest.raises(ValueError, match=r"t\.total"):
|
|
197
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def test_stddev_without_distribution_is_rejected():
|
|
201
|
+
tables = _single_column_table(ColumnDef("total", "numeric", note={"stddev": 5}))
|
|
202
|
+
with pytest.raises(
|
|
203
|
+
ValueError, match=r't\.total: "stddev" only applies with "distribution": "normal"\.'
|
|
204
|
+
):
|
|
205
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def test_stddev_with_the_wrong_distribution_is_rejected():
|
|
209
|
+
tables = _single_column_table(
|
|
210
|
+
ColumnDef("total", "numeric", note={"distribution": "exponential", "stddev": 5})
|
|
211
|
+
)
|
|
212
|
+
with pytest.raises(ValueError, match=r"t\.total"):
|
|
213
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def test_median_with_the_wrong_distribution_is_rejected():
|
|
217
|
+
tables = _single_column_table(
|
|
218
|
+
ColumnDef("total", "numeric", note={"distribution": "normal", "median": 5})
|
|
219
|
+
)
|
|
220
|
+
with pytest.raises(ValueError, match=r"t\.total"):
|
|
221
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def test_spread_with_the_wrong_distribution_is_rejected():
|
|
225
|
+
tables = _single_column_table(
|
|
226
|
+
ColumnDef("total", "numeric", note={"distribution": "normal", "spread": 0.5})
|
|
227
|
+
)
|
|
228
|
+
with pytest.raises(ValueError, match=r"t\.total"):
|
|
229
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def test_mean_without_a_matching_distribution_is_rejected():
|
|
233
|
+
tables = _single_column_table(ColumnDef("total", "numeric", note={"mean": 5}))
|
|
234
|
+
with pytest.raises(ValueError, match=r"t\.total"):
|
|
235
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
@pytest.mark.parametrize(
|
|
239
|
+
"note",
|
|
240
|
+
[
|
|
241
|
+
{"distribution": "normal", "stddev": -1},
|
|
242
|
+
{"distribution": "normal", "stddev": 0},
|
|
243
|
+
{"distribution": "lognormal", "median": -5},
|
|
244
|
+
{"distribution": "lognormal", "spread": 0},
|
|
245
|
+
],
|
|
246
|
+
)
|
|
247
|
+
def test_non_positive_stddev_median_spread_is_rejected(note):
|
|
248
|
+
tables = _single_column_table(ColumnDef("total", "numeric", note=note))
|
|
249
|
+
with pytest.raises(ValueError, match=r"t\.total"):
|
|
250
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def test_a_bool_value_is_rejected_where_a_number_is_expected():
|
|
254
|
+
tables = _single_column_table(
|
|
255
|
+
ColumnDef("total", "numeric", note={"distribution": "normal", "mean": True})
|
|
256
|
+
)
|
|
257
|
+
with pytest.raises(ValueError, match=r"t\.total"):
|
|
258
|
+
generate_data_from_dbml(tables, [], base_rows=5, seed=1)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{model2data-1.6.0 → model2data-1.7.0}/model2data/dbt/templates/macros/generate_schema_name.sql
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|