model2data 1.6.0__tar.gz → 1.7.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. {model2data-1.6.0/model2data.egg-info → model2data-1.7.1}/PKG-INFO +1 -1
  2. {model2data-1.6.0 → model2data-1.7.1}/README.md +24 -1
  3. {model2data-1.6.0 → model2data-1.7.1}/model2data/generate/core.py +33 -0
  4. {model2data-1.6.0 → model2data-1.7.1}/model2data/generate/faker.py +176 -10
  5. {model2data-1.6.0 → model2data-1.7.1}/model2data/generate/hints.py +73 -4
  6. {model2data-1.6.0 → model2data-1.7.1/model2data.egg-info}/PKG-INFO +1 -1
  7. {model2data-1.6.0 → model2data-1.7.1}/model2data.egg-info/SOURCES.txt +2 -0
  8. {model2data-1.6.0 → model2data-1.7.1}/pyproject.toml +1 -1
  9. model2data-1.7.1/tests/test_distributions.py +258 -0
  10. model2data-1.7.1/tests/test_lone_country.py +132 -0
  11. {model2data-1.6.0 → model2data-1.7.1}/LICENSE +0 -0
  12. {model2data-1.6.0 → model2data-1.7.1}/README_PYPI.md +0 -0
  13. {model2data-1.6.0 → model2data-1.7.1}/model2data/__init__.py +0 -0
  14. {model2data-1.6.0 → model2data-1.7.1}/model2data/cli.py +0 -0
  15. {model2data-1.6.0 → model2data-1.7.1}/model2data/dbt/__init__.py +0 -0
  16. {model2data-1.6.0 → model2data-1.7.1}/model2data/dbt/project.py +0 -0
  17. {model2data-1.6.0 → model2data-1.7.1}/model2data/dbt/templates/dbt_project.yml.jinja +0 -0
  18. {model2data-1.6.0 → model2data-1.7.1}/model2data/dbt/templates/macros/generate_schema_name.sql +0 -0
  19. {model2data-1.6.0 → model2data-1.7.1}/model2data/dbt/templates/profiles.yml.jinja +0 -0
  20. {model2data-1.6.0 → model2data-1.7.1}/model2data/dbt/tests.py +0 -0
  21. {model2data-1.6.0 → model2data-1.7.1}/model2data/generate/__init__.py +0 -0
  22. {model2data-1.6.0 → model2data-1.7.1}/model2data/generate/options.py +0 -0
  23. {model2data-1.6.0 → model2data-1.7.1}/model2data/generate/relationships.py +0 -0
  24. {model2data-1.6.0 → model2data-1.7.1}/model2data/generate/timeline.py +0 -0
  25. {model2data-1.6.0 → model2data-1.7.1}/model2data/parse/__init__.py +0 -0
  26. {model2data-1.6.0 → model2data-1.7.1}/model2data/parse/dbml.py +0 -0
  27. {model2data-1.6.0 → model2data-1.7.1}/model2data/utils.py +0 -0
  28. {model2data-1.6.0 → model2data-1.7.1}/model2data.egg-info/dependency_links.txt +0 -0
  29. {model2data-1.6.0 → model2data-1.7.1}/model2data.egg-info/entry_points.txt +0 -0
  30. {model2data-1.6.0 → model2data-1.7.1}/model2data.egg-info/requires.txt +0 -0
  31. {model2data-1.6.0 → model2data-1.7.1}/model2data.egg-info/top_level.txt +0 -0
  32. {model2data-1.6.0 → model2data-1.7.1}/setup.cfg +0 -0
  33. {model2data-1.6.0 → model2data-1.7.1}/tests/test_as_of_anchor.py +0 -0
  34. {model2data-1.6.0 → model2data-1.7.1}/tests/test_cli.py +0 -0
  35. {model2data-1.6.0 → model2data-1.7.1}/tests/test_column_time_hints.py +0 -0
  36. {model2data-1.6.0 → model2data-1.7.1}/tests/test_coverage_gaps.py +0 -0
  37. {model2data-1.6.0 → model2data-1.7.1}/tests/test_dbml_parser.py +0 -0
  38. {model2data-1.6.0 → model2data-1.7.1}/tests/test_dbml_parser_fuzz.py +0 -0
  39. {model2data-1.6.0 → model2data-1.7.1}/tests/test_dbt_integration.py +0 -0
  40. {model2data-1.6.0 → model2data-1.7.1}/tests/test_dbt_naming.py +0 -0
  41. {model2data-1.6.0 → model2data-1.7.1}/tests/test_dbt_project.py +0 -0
  42. {model2data-1.6.0 → model2data-1.7.1}/tests/test_dbt_tests.py +0 -0
  43. {model2data-1.6.0 → model2data-1.7.1}/tests/test_faker_name_inference.py +0 -0
  44. {model2data-1.6.0 → model2data-1.7.1}/tests/test_generation.py +0 -0
  45. {model2data-1.6.0 → model2data-1.7.1}/tests/test_options.py +0 -0
  46. {model2data-1.6.0 → model2data-1.7.1}/tests/test_release_stress.py +0 -0
  47. {model2data-1.6.0 → model2data-1.7.1}/tests/test_row_identity.py +0 -0
  48. {model2data-1.6.0 → model2data-1.7.1}/tests/test_shaping.py +0 -0
  49. {model2data-1.6.0 → model2data-1.7.1}/tests/test_table_seeds.py +0 -0
  50. {model2data-1.6.0 → model2data-1.7.1}/tests/test_timeline.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: model2data
3
- Version: 1.6.0
3
+ Version: 1.7.1
4
4
  Summary: Generate analytics-ready datasets from DBML models
5
5
  Author: JB Analytica
6
6
  License-Expression: MIT
@@ -159,7 +159,10 @@ model2data --file examples/ecommerce.dbml --rows 200 --seed 42 --table-seed orde
159
159
  ```
160
160
 
161
161
  `--locale` picks the country every generated person and address comes from (`en_US` by default);
162
- it's a per-run setting, so a table can't end up holding one Belgian and one American address:
162
+ it's a per-run setting, so a table can't end up holding one Belgian and one American address. A
163
+ `country` column that sits beside a `city`/`street`/`state`/`postcode` column always agrees with
164
+ that place; a `country` column with none of those beside it isn't describing anyone's address, so
165
+ it reads as an international mix instead, with the locale's own country the most common:
163
166
 
164
167
  ```bash
165
168
  model2data --file examples/ecommerce.dbml --rows 200 --seed 42 --locale nl_BE
@@ -232,6 +235,26 @@ non-null rows a boolean column comes back `true`. `null_rate` replaces the colum
232
235
  fraction outright. `distinct` draws the column's values from a fixed-size pool instead of a fresh
233
236
  value per row — a `shipping_city` most warehouses only ever see a handful of.
234
237
 
238
+ ### Shape a number's distribution
239
+
240
+ `min`/`max` alone only ever drew uniformly between them. A `distribution` note hint on an
241
+ integer or decimal column picks a different shape instead:
242
+
243
+ ```dbml
244
+ Table orders {
245
+ id int [pk]
246
+ total_amount numeric [note: '{"distribution": "lognormal", "median": 80, "spread": 0.6, "min": 5}']
247
+ }
248
+ ```
249
+
250
+ `normal` takes `mean` (the centre) and `stddev` (the spread); `lognormal` takes `median` (the
251
+ typical value) and `spread` (how heavy the tail is — 0.3 is mild, 1.0 is heavy); `exponential`
252
+ takes `mean` (the average). Any left unset default to the midpoint of the column's effective
253
+ `min`/`max` (or `stddev` = range / 6, `spread` = 0.5). `min`/`max` still clip the result — a
254
+ `normal` centred near an edge redraws a bounded number of times before clamping, so it never
255
+ loops forever and never crosses the bound. Leaving `distribution` out, or setting it to
256
+ `"uniform"`, is exactly today's behaviour.
257
+
235
258
  Run dbt to load, transform, and test the data:
236
259
 
237
260
  ```bash
@@ -15,6 +15,7 @@ from model2data.generate.faker import (
15
15
  release_row_pools,
16
16
  reset_duplicate_unique_columns,
17
17
  reset_row_pools,
18
+ resolve_address_pool_field,
18
19
  set_locale,
19
20
  )
20
21
  from model2data.generate.hints import validate_hints
@@ -201,6 +202,12 @@ def generate_data_from_dbml(
201
202
  for column_name in key.get("columns") or []
202
203
  }
203
204
 
205
+ # A table's own shape, worked out before a single value is drawn: does
206
+ # this table have a `country` column with no `city`/`street`/`state`/
207
+ # `postcode` beside it to keep coherent with. See
208
+ # _lone_country_columns.
209
+ lone_country_columns = _lone_country_columns(table_def)
210
+
204
211
  # -----------------------
205
212
  # First pass: columns + FKs
206
213
  # -----------------------
@@ -230,6 +237,7 @@ def generate_data_from_dbml(
230
237
  as_of=as_of,
231
238
  time_profile=profile,
232
239
  skew=skew,
240
+ lone_country=column.name in lone_country_columns,
233
241
  )
234
242
 
235
243
  df = pd.DataFrame(data)
@@ -303,6 +311,31 @@ def generate_data_from_dbml(
303
311
  # ---------------------------------------------------------
304
312
  # Internal helpers
305
313
  # ---------------------------------------------------------
314
+ def _lone_country_columns(table_def: TableDef) -> set[str]:
315
+ """Names of this table's *lone* country columns.
316
+
317
+ A `country` column reads as the locale's own country on every row when it
318
+ sits beside a `city`/`street`/`state`/`postcode` column -- together they
319
+ describe one place, and the country has to agree with the rest of it.
320
+ Alone, repeating that same country on every row reads as a single-country
321
+ customer base rather than an international one, so
322
+ `generate_column_values` draws it from a home-heavy mix instead (see
323
+ `faker._HOME_COUNTRY_SHARE`). `country` columns don't count as company
324
+ for each other -- only a *different* address-pool field does.
325
+ """
326
+ address_fields = {
327
+ column.name: field
328
+ for column in table_def.columns
329
+ for field in [resolve_address_pool_field(column)]
330
+ if field is not None
331
+ }
332
+ country_columns = {name for name, field in address_fields.items() if field == "country"}
333
+ if not country_columns:
334
+ return set()
335
+ has_place_column = any(field != "country" for field in address_fields.values())
336
+ return set() if has_place_column else country_columns
337
+
338
+
306
339
  def _coerce_integer_dtypes(df: pd.DataFrame, table_def: TableDef) -> pd.DataFrame:
307
340
  """
308
341
  Cast int/bigint/smallint-typed columns to pandas' nullable "Int64" dtype.
@@ -505,6 +505,42 @@ def _infer_by_type(base_type: str) -> Optional[_Provider]:
505
505
  return lambda: fake.format(base_type)
506
506
 
507
507
 
508
+ def resolve_address_pool_field(column: ColumnDef) -> Optional[str]:
509
+ """The address-pool field (`street`, `full`, `city`, `state`, `postcode`,
510
+ `country`) this column would draw from, if any -- else None.
511
+
512
+ Same declared-type-then-name precedence `generate_column_values`'s own
513
+ untyped-column branch uses (`_infer_by_type` before `_infer_by_name`),
514
+ narrowed to the address pool. Exposed for `generate.core` to tell a
515
+ *lone* country column -- the only address-pool-shaped column in its
516
+ table -- from one that sits beside a `city`/`street`/`state`/`postcode`
517
+ column, before a single value of the table has been generated.
518
+
519
+ Restricted to columns that would actually reach that branch: an enum
520
+ column, or one whose type is a structured (int/date/uuid/...) type, never
521
+ gets there in `generate_column_values` -- and `_infer_by_type` probes an
522
+ unrecognized type by actually calling it (`fake.format(base_type)`), so
523
+ running it over a `date` or `int` column here would consume real draws
524
+ from the shared RNG and shift every value generated after it, breaking
525
+ reproducibility for reasons invisible to whoever hits it.
526
+ """
527
+ if column.enum_values or not is_free_text_type(column.data_type):
528
+ return None
529
+ base_type = column.data_type.lower().split("(")[0].strip()
530
+ generator = _infer_by_type(base_type) or _infer_by_name(column.name)
531
+ if isinstance(generator, _FromRow) and generator.pool == "address":
532
+ return generator.field
533
+ return None
534
+
535
+
536
+ # A lone country column (see resolve_address_pool_field's caller) mixes the
537
+ # locale's own country in with the rest of the world rather than repeating it
538
+ # on every row. 0.6 is a default, not a claim about any real market -- a
539
+ # business selling internationally still has a home market, and most of its
540
+ # rows are plausibly it, but "most" is not "all".
541
+ _HOME_COUNTRY_SHARE = 0.6
542
+
543
+
508
544
  def _column_time_profile(
509
545
  column: ColumnDef, time_profile: Optional[TimeProfile]
510
546
  ) -> Optional[TimeProfile]:
@@ -580,6 +616,7 @@ def generate_column_values(
580
616
  as_of: AsOf = None,
581
617
  time_profile: Optional[TimeProfile] = None,
582
618
  skew: float = 0.0,
619
+ lone_country: bool = False,
583
620
  ) -> list:
584
621
  """
585
622
  Generate synthetic values for a single column.
@@ -591,6 +628,16 @@ def generate_column_values(
591
628
  every path that draws a value -- the main pass, the self-referencing FK
592
629
  repair, the composite-key retry -- draws it the same way.
593
630
 
631
+ `lone_country` tells the address-pool branch this column is the *only*
632
+ address-shaped column in its table (see
633
+ `generate.core._lone_country_columns`). A `country` column that sits
634
+ beside a `city`/`street`/`state`/`postcode` column still reads that
635
+ place's own country, byte-identical to earlier releases; a lone one
636
+ instead draws a home-heavy mix of the locale's country and the wider
637
+ world, since "Belgium" on every row of a customers table with no other
638
+ address column reads as a single-country customer base rather than an
639
+ international one.
640
+
594
641
  `as_of` is the date every generated date and timestamp is placed relative
595
642
  to, defaulting to today. Pass it to make a seeded run reproduce on any
596
643
  later day rather than only on the day it first ran.
@@ -629,6 +676,7 @@ def generate_column_values(
629
676
  as_of=as_of,
630
677
  time_profile=time_profile,
631
678
  skew=skew,
679
+ lone_country=lone_country,
632
680
  )
633
681
  values = random.choices(pool, k=row_count)
634
682
  if not force_not_null and "not null" not in column.settings and "pk" not in column.settings:
@@ -665,6 +713,13 @@ def generate_column_values(
665
713
  if column.note:
666
714
  min_val = column.note.get("min")
667
715
  max_val = column.note.get("max")
716
+ # The note's own min/max, before either numeric branch fills in its
717
+ # branch-specific default -- these are the only bounds a distribution
718
+ # hint clips against (see `_clipped_distribution_draw`). A column with no
719
+ # explicit min/max draws from an unclipped distribution even though the
720
+ # branch below still has an implicit default range for the *parameters*.
721
+ explicit_min, explicit_max = min_val, max_val
722
+ distribution = column_note.get("distribution", "uniform")
668
723
 
669
724
  if fk_series is not None and not fk_series.empty:
670
725
  # A plain branch of the same if/elif chain (rather than an early
@@ -722,7 +777,22 @@ def generate_column_values(
722
777
  if max_val is None:
723
778
  max_val = 100
724
779
 
725
- if ensure_unique:
780
+ if distribution != "uniform":
781
+ # A shaped column never needs the row_count-sized value space
782
+ # random.sample relies on below, so it always draws one value at
783
+ # a time -- unique or not -- and leans on _deduplicate the same
784
+ # way every other non-uniform-fast-path branch does.
785
+ params = _distribution_params(distribution, min_val, max_val, column_note)
786
+
787
+ def generator() -> int:
788
+ return round(
789
+ _clipped_distribution_draw(distribution, params, explicit_min, explicit_max)
790
+ )
791
+
792
+ values = [generator() for _ in range(row_count)]
793
+ if ensure_unique:
794
+ values = _deduplicate(values, generator, column_name=unique_label)
795
+ elif ensure_unique:
726
796
  if not had_explicit_range:
727
797
  # No user-specified range: widen the default so there's
728
798
  # always enough headroom for `row_count` unique PK values.
@@ -753,13 +823,27 @@ def generate_column_values(
753
823
  min_val = 0
754
824
  if max_val is None:
755
825
  max_val = 10_000
756
- values = [round(random.uniform(min_val, max_val), 2) for _ in range(row_count)]
757
- if ensure_unique:
758
- values = _deduplicate(
759
- values,
760
- lambda: round(random.uniform(min_val, max_val), 2),
761
- column_name=unique_label,
762
- )
826
+
827
+ if distribution != "uniform":
828
+ params = _distribution_params(distribution, min_val, max_val, column_note)
829
+
830
+ def generator() -> float:
831
+ return round(
832
+ _clipped_distribution_draw(distribution, params, explicit_min, explicit_max),
833
+ 2,
834
+ )
835
+
836
+ values = [generator() for _ in range(row_count)]
837
+ if ensure_unique:
838
+ values = _deduplicate(values, generator, column_name=unique_label)
839
+ else:
840
+ values = [round(random.uniform(min_val, max_val), 2) for _ in range(row_count)]
841
+ if ensure_unique:
842
+ values = _deduplicate(
843
+ values,
844
+ lambda: round(random.uniform(min_val, max_val), 2),
845
+ column_name=unique_label,
846
+ )
763
847
 
764
848
  # -----------------------------------------------------
765
849
  # Booleans
@@ -791,8 +875,18 @@ def generate_column_values(
791
875
  else:
792
876
  generator = _infer_by_type(base_type) or _infer_by_name(column.name)
793
877
  if isinstance(generator, _FromRow):
794
- rows = _row_pool(generator.pool, table_name, row_count)
795
- values = [getattr(rows[index], generator.field) for index in range(row_count)]
878
+ if lone_country and generator.pool == "address" and generator.field == "country":
879
+ # No sibling city/street/state/postcode column to keep this
880
+ # one coherent with, so it isn't "this row's place" at all --
881
+ # draw a home-heavy mix instead of repeating the locale's own
882
+ # country on every row.
883
+ values = [
884
+ _country_name if random.random() < _HOME_COUNTRY_SHARE else fake.country()
885
+ for _ in range(row_count)
886
+ ]
887
+ else:
888
+ rows = _row_pool(generator.pool, table_name, row_count)
889
+ values = [getattr(rows[index], generator.field) for index in range(row_count)]
796
890
  if ensure_unique:
797
891
  values = _deduplicate_identity(values)
798
892
  elif generator is not None:
@@ -824,6 +918,78 @@ def generate_column_values(
824
918
  # ---------------------------------------------------------
825
919
  # Internal helpers
826
920
  # ---------------------------------------------------------
921
+ # How many times a clipped draw redraws before giving up and clamping. Kept
922
+ # small and fixed rather than exposed as a hint: a normal whose mean sits
923
+ # well inside its min/max rarely needs a redraw at all, and a distribution
924
+ # whose parameters put most of its mass outside the bounds is a schema
925
+ # mistake no retry count fixes -- clamping is the honest fallback either way.
926
+ _MAX_DISTRIBUTION_REDRAWS = 20
927
+
928
+
929
+ def _distribution_params(
930
+ distribution: str, effective_min: float, effective_max: float, note: dict
931
+ ) -> dict:
932
+ """Fill in whichever of mean/stddev/median/spread `note` leaves unset.
933
+
934
+ `effective_min`/`effective_max` are the note's own `min`/`max` when given,
935
+ else the calling branch's default range (0-100 for integers, 0-10,000 for
936
+ decimals) -- ordinary enough a midpoint to centre an unparameterized
937
+ distribution on.
938
+ """
939
+ midpoint = (effective_min + effective_max) / 2
940
+ if distribution == "normal":
941
+ return {
942
+ "mean": note.get("mean", midpoint),
943
+ "stddev": note.get("stddev", (effective_max - effective_min) / 6),
944
+ }
945
+ if distribution == "lognormal":
946
+ return {
947
+ "median": note.get("median", midpoint),
948
+ "spread": note.get("spread", 0.5),
949
+ }
950
+ # exponential: hints.py has already rejected anything else reaching here.
951
+ return {"mean": note.get("mean", midpoint)}
952
+
953
+
954
+ def _draw_distribution_value(distribution: str, params: dict) -> float:
955
+ """One raw draw from `distribution`, unclipped and unrounded."""
956
+ if distribution == "normal":
957
+ return random.gauss(params["mean"], params["stddev"])
958
+ if distribution == "lognormal":
959
+ return random.lognormvariate(math.log(params["median"]), params["spread"])
960
+ return random.expovariate(1 / params["mean"]) # exponential
961
+
962
+
963
+ def _clipped_distribution_draw(
964
+ distribution: str,
965
+ params: dict,
966
+ clip_min: Optional[float],
967
+ clip_max: Optional[float],
968
+ ) -> float:
969
+ """Draw from `distribution`, redrawing out-of-bounds values before clamping.
970
+
971
+ A normal centred well inside [min, max] almost never needs the clamp; one
972
+ centred near an edge (mean 120, min 0) would otherwise pile values up at
973
+ the boundary, so a bounded number of redraws is tried first and only a
974
+ value still out of bounds after all of them gets clamped -- which also
975
+ keeps this from looping forever when min and max leave no room at all.
976
+ `clip_min`/`clip_max` are the note's own bounds (`None` when the column
977
+ didn't set one), never the branch's implicit default range.
978
+ """
979
+ value = _draw_distribution_value(distribution, params)
980
+ attempts = 0
981
+ while attempts < _MAX_DISTRIBUTION_REDRAWS and (
982
+ (clip_min is not None and value < clip_min) or (clip_max is not None and value > clip_max)
983
+ ):
984
+ value = _draw_distribution_value(distribution, params)
985
+ attempts += 1
986
+ if clip_min is not None and value < clip_min:
987
+ value = clip_min
988
+ if clip_max is not None and value > clip_max:
989
+ value = clip_max
990
+ return value
991
+
992
+
827
993
  def _null_fraction_for(column: ColumnDef, row_count: int) -> float:
828
994
  """The fraction of `row_count` rows this column should turn null.
829
995
 
@@ -3,10 +3,11 @@
3
3
  A note has always been either plain text (a comment, ignored by generation)
4
4
  or a JSON object read for `min`/`max`. This module documents the rest of that
5
5
  object's vocabulary -- `null_rate`, `weights`, `true_rate`, `distinct`, `skew`,
6
- `after`, `business_hours`, `growth`, `seasonality` -- and checks it once,
7
- before a single row is generated, so a typo'd enum value or a hint on the
8
- wrong kind of column fails with a message naming the table and column rather
9
- than surfacing as a wrong-looking dataset or a downstream dbt test failure.
6
+ `after`, `business_hours`, `growth`, `seasonality`, `distribution` and its
7
+ parameters -- and checks it once, before a single row is generated, so a
8
+ typo'd enum value or a hint on the wrong kind of column fails with a message
9
+ naming the table and column rather than surfacing as a wrong-looking dataset
10
+ or a downstream dbt test failure.
10
11
 
11
12
  | key | applies to | meaning |
12
13
  |-------------|-----------------------------------------------|--------------------------------------------|
@@ -20,6 +21,11 @@ than surfacing as a wrong-looking dataset or a downstream dbt test failure.
20
21
  | `business_hours` | date/timestamp columns | overrides the run-level `TimeProfile.business_hours` for this column |
21
22
  | `growth` | date/timestamp columns | overrides the run-level `TimeProfile.growth` for this column |
22
23
  | `seasonality` | date/timestamp columns | overrides the run-level `TimeProfile.seasonality` for this column |
24
+ | `distribution` | numeric columns | draw shape: `uniform` (default), `normal`, `lognormal`, `exponential` |
25
+ | `mean` | numeric columns, with `distribution: normal` or `exponential` | normal: the centre; exponential: the average (scale) |
26
+ | `stddev` | numeric columns, with `distribution: normal` | normal: the spread |
27
+ | `median` | numeric columns, with `distribution: lognormal`| lognormal: the typical value, `exp(mu)` |
28
+ | `spread` | numeric columns, with `distribution: lognormal`| lognormal: sigma of the underlying normal (0.3 mild, 1.0 heavy tail) |
23
29
  """
24
30
 
25
31
  from __future__ import annotations
@@ -95,6 +101,33 @@ def _is_temporal_type(base_type: str) -> bool:
95
101
  return any(key in base_type for key in ("timestamp", "datetime"))
96
102
 
97
103
 
104
+ def _is_numeric_type(base_type: str) -> bool:
105
+ """True for the integer and float/decimal types `min`/`max`/`distribution` apply to.
106
+
107
+ Mirrors generate.faker's own two numeric branches exactly, so a hint this
108
+ module accepts is guaranteed to land on a branch that reads it.
109
+ """
110
+ return any(
111
+ key in base_type
112
+ for key in ("int", "integer", "bigint", "smallint", "decimal", "numeric", "float", "double")
113
+ )
114
+
115
+
116
+ # Distributions a numeric column's `distribution` hint may name.
117
+ _DISTRIBUTIONS = ("uniform", "normal", "lognormal", "exponential")
118
+
119
+ # Which distribution(s) each shape parameter is meaningful under -- checked
120
+ # against the note's own `distribution` (default "uniform" when absent), so
121
+ # a `stddev` with no `distribution` key is rejected the same as one paired
122
+ # with the wrong distribution.
123
+ _PARAM_DISTRIBUTIONS: dict[str, tuple[str, ...]] = {
124
+ "mean": ("normal", "exponential"),
125
+ "stddev": ("normal",),
126
+ "median": ("lognormal",),
127
+ "spread": ("lognormal",),
128
+ }
129
+
130
+
98
131
  def _validate_column_hints(
99
132
  table_name: str,
100
133
  column: ColumnDef,
@@ -110,6 +143,7 @@ def _validate_column_hints(
110
143
  is_enum = bool(column.enum_values)
111
144
  is_boolean = "boolean" in base_type or "bool" in base_type
112
145
  is_temporal = _is_temporal_type(base_type)
146
+ is_numeric = _is_numeric_type(base_type)
113
147
  is_nullable = not is_pk and "not null" not in column.settings
114
148
 
115
149
  if "null_rate" in note:
@@ -170,6 +204,31 @@ def _validate_column_hints(
170
204
  raise ValueError(f'{label}: "seasonality" only applies to date/timestamp columns.')
171
205
  _check_fraction(label, "seasonality", note["seasonality"])
172
206
 
207
+ if "distribution" in note:
208
+ if not is_numeric:
209
+ raise ValueError(f'{label}: "distribution" only applies to numeric columns.')
210
+ if note["distribution"] not in _DISTRIBUTIONS:
211
+ allowed = ", ".join(f'"{d}"' for d in _DISTRIBUTIONS)
212
+ raise ValueError(
213
+ f'{label}: "distribution" must be one of {allowed} (got {note["distribution"]!r}).'
214
+ )
215
+
216
+ # mean/stddev/median/spread only make sense alongside the distribution
217
+ # they shape, so each is checked against the note's own `distribution`
218
+ # (absent means "uniform", which none of them apply to either).
219
+ for param_key, required_distributions in _PARAM_DISTRIBUTIONS.items():
220
+ if param_key not in note:
221
+ continue
222
+ if not is_numeric:
223
+ raise ValueError(f'{label}: "{param_key}" only applies to numeric columns.')
224
+ if note.get("distribution") not in required_distributions:
225
+ options = " or ".join(f'"{d}"' for d in required_distributions)
226
+ raise ValueError(f'{label}: "{param_key}" only applies with "distribution": {options}.')
227
+ if param_key == "mean":
228
+ _check_number(label, param_key, note[param_key])
229
+ else:
230
+ _check_positive_number(label, param_key, note[param_key])
231
+
173
232
 
174
233
  def _check_fraction(label: str, key: str, value: object) -> None:
175
234
  if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0.0 <= value <= 1.0:
@@ -181,6 +240,16 @@ def _check_positive_int(label: str, key: str, value: object) -> None:
181
240
  raise ValueError(f'{label}: "{key}" must be a positive whole number (got {value!r}).')
182
241
 
183
242
 
243
+ def _check_number(label: str, key: str, value: object) -> None:
244
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
245
+ raise ValueError(f'{label}: "{key}" must be a number (got {value!r}).')
246
+
247
+
248
+ def _check_positive_number(label: str, key: str, value: object) -> None:
249
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
250
+ raise ValueError(f'{label}: "{key}" must be a number greater than 0 (got {value!r}).')
251
+
252
+
184
253
  def _check_bool(label: str, key: str, value: object) -> None:
185
254
  if not isinstance(value, bool):
186
255
  raise ValueError(f'{label}: "{key}" must be true or false (got {value!r}).')
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: model2data
3
- Version: 1.6.0
3
+ Version: 1.7.1
4
4
  Summary: Generate analytics-ready datasets from DBML models
5
5
  Author: JB Analytica
6
6
  License-Expression: MIT
@@ -36,8 +36,10 @@ tests/test_dbt_integration.py
36
36
  tests/test_dbt_naming.py
37
37
  tests/test_dbt_project.py
38
38
  tests/test_dbt_tests.py
39
+ tests/test_distributions.py
39
40
  tests/test_faker_name_inference.py
40
41
  tests/test_generation.py
42
+ tests/test_lone_country.py
41
43
  tests/test_options.py
42
44
  tests/test_release_stress.py
43
45
  tests/test_row_identity.py
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "model2data"
7
- version = "1.6.0"
7
+ version = "1.7.1"
8
8
  description = "Generate analytics-ready datasets from DBML models"
9
9
  readme = "README_PYPI.md"
10
10
  requires-python = ">=3.10"
@@ -0,0 +1,258 @@
1
+ """Numeric distribution hints: `distribution` and its `mean`/`stddev`/`median`/`spread`
2
+ parameters on integer and decimal columns.
3
+
4
+ The product ask this closes was "can I make a normal distribution or another kind
5
+ of distribution as well?" on a numeric column -- `min`/`max` alone could only shape
6
+ a uniform spread. Every statistical test here uses a fixed seed and a generous
7
+ tolerance so it never flakes; the exact bounds were chosen by running the real
8
+ implementation and leaving headroom, not derived analytically.
9
+ """
10
+
11
+ import statistics
12
+
13
+ import pytest
14
+
15
+ from model2data.generate.core import generate_data_from_dbml
16
+ from model2data.parse.dbml import ColumnDef, TableDef
17
+
18
+
19
+ def _single_column_table(column: ColumnDef, table_name: str = "t") -> dict[str, TableDef]:
20
+ return {table_name: TableDef(name=table_name, columns=[ColumnDef("id", "int", {"pk"}), column])}
21
+
22
+
23
+ # ---------------------------------------------------------
24
+ # Step 0: pin today's (uniform) behaviour before touching it
25
+ # ---------------------------------------------------------
26
+ def test_no_distribution_hint_reproduces_pre_1_7_frames():
27
+ """A numeric column with no `distribution` key must still produce exactly
28
+ what 1.6.0 produced with the same seed -- the guard rail every distribution
29
+ added here works against."""
30
+ tables = {
31
+ "t": TableDef(
32
+ name="t",
33
+ columns=[
34
+ ColumnDef("id", "int", {"pk"}),
35
+ ColumnDef("amount", "numeric", {"not null"}),
36
+ ColumnDef("qty", "int", {"not null"}),
37
+ ],
38
+ )
39
+ }
40
+ data = generate_data_from_dbml(tables, [], base_rows=8, seed=777)
41
+ assert data["t"]["amount"].tolist() == [
42
+ 4169.09,
43
+ 3848.52,
44
+ 5163.13,
45
+ 3006.66,
46
+ 25.14,
47
+ 4363.61,
48
+ 685.34,
49
+ 8357.29,
50
+ ]
51
+ assert data["t"]["qty"].tolist() == [7, 89, 51, 52, 82, 100, 97, 9]
52
+
53
+
54
+ # ---------------------------------------------------------
55
+ # Shape: each distribution draws roughly what it says it does
56
+ # ---------------------------------------------------------
57
+ def test_normal_distribution_matches_mean_and_stddev():
58
+ tables = _single_column_table(
59
+ ColumnDef(
60
+ "amount",
61
+ "numeric",
62
+ {"not null"},
63
+ note={"distribution": "normal", "mean": 100, "stddev": 10},
64
+ )
65
+ )
66
+ data = generate_data_from_dbml(tables, [], base_rows=5000, seed=1)
67
+ values = data["t"]["amount"].tolist()
68
+ assert 97 <= statistics.mean(values) <= 103
69
+ assert 8 <= statistics.stdev(values) <= 12
70
+
71
+
72
+ def test_lognormal_distribution_median_and_mean():
73
+ tables = _single_column_table(
74
+ ColumnDef(
75
+ "amount",
76
+ "numeric",
77
+ {"not null"},
78
+ note={"distribution": "lognormal", "median": 80, "spread": 0.5},
79
+ )
80
+ )
81
+ data = generate_data_from_dbml(tables, [], base_rows=5000, seed=2)
82
+ values = data["t"]["amount"].tolist()
83
+ sample_median = statistics.median(values)
84
+ assert 70 <= sample_median <= 90
85
+ assert statistics.mean(values) > sample_median
86
+
87
+
88
+ def test_exponential_distribution_matches_mean():
89
+ tables = _single_column_table(
90
+ ColumnDef(
91
+ "amount", "numeric", {"not null"}, note={"distribution": "exponential", "mean": 30}
92
+ )
93
+ )
94
+ data = generate_data_from_dbml(tables, [], base_rows=5000, seed=3)
95
+ values = data["t"]["amount"].tolist()
96
+ assert 26 <= statistics.mean(values) <= 34
97
+
98
+
99
+ def test_missing_parameters_default_to_midpoint_and_derived_spread():
100
+ """No `mean`/`stddev` at all: normal centres on the midpoint of the
101
+ default int range (0-100 -> 50) with stddev (max-min)/6."""
102
+ tables = _single_column_table(
103
+ ColumnDef("qty", "int", {"not null"}, note={"distribution": "normal"})
104
+ )
105
+ data = generate_data_from_dbml(tables, [], base_rows=5000, seed=7)
106
+ values = data["t"]["qty"].tolist()
107
+ assert 45 <= statistics.mean(values) <= 55
108
+ assert 12 <= statistics.stdev(values) <= 21
109
+
110
+
111
+ def test_missing_lognormal_parameters_default_to_midpoint_and_half_spread():
112
+ """No `median`/`spread`: lognormal centres on the midpoint of the default
113
+ decimal range (0-10,000 -> 5,000)."""
114
+ tables = _single_column_table(
115
+ ColumnDef("amount", "numeric", {"not null"}, note={"distribution": "lognormal"})
116
+ )
117
+ data = generate_data_from_dbml(tables, [], base_rows=5000, seed=8)
118
+ values = data["t"]["amount"].tolist()
119
+ assert 4000 <= statistics.median(values) <= 6000
120
+
121
+
122
+ def test_missing_exponential_mean_defaults_to_midpoint():
123
+ """No `mean`: exponential's scale defaults to the midpoint of the default
124
+ int range (0-100 -> 50)."""
125
+ tables = _single_column_table(
126
+ ColumnDef("qty", "int", {"not null"}, note={"distribution": "exponential"})
127
+ )
128
+ data = generate_data_from_dbml(tables, [], base_rows=5000, seed=9)
129
+ values = data["t"]["qty"].tolist()
130
+ assert 40 <= statistics.mean(values) <= 60
131
+
132
+
133
+ # ---------------------------------------------------------
134
+ # Clipping and typing
135
+ # ---------------------------------------------------------
136
+ def test_clipping_with_explicit_min_leaves_no_negatives():
137
+ """A normal whose mean sits close to an explicit `min` would otherwise
138
+ draw negatives; redraw-then-clamp keeps every value in bounds."""
139
+ tables = _single_column_table(
140
+ ColumnDef(
141
+ "amount",
142
+ "numeric",
143
+ {"not null"},
144
+ note={"distribution": "normal", "mean": 5, "stddev": 10, "min": 0},
145
+ )
146
+ )
147
+ data = generate_data_from_dbml(tables, [], base_rows=2000, seed=4)
148
+ values = data["t"]["amount"].tolist()
149
+ assert min(values) >= 0
150
+
151
+
152
+ def test_integer_columns_stay_ints_under_a_distribution():
153
+ tables = _single_column_table(
154
+ ColumnDef(
155
+ "qty", "int", {"not null"}, note={"distribution": "normal", "mean": 50, "stddev": 10}
156
+ )
157
+ )
158
+ data = generate_data_from_dbml(tables, [], base_rows=200, seed=5)
159
+ values = data["t"]["qty"].tolist()
160
+ assert all(isinstance(v, int) for v in values)
161
+
162
+
163
+ def test_unique_normal_column_stays_unique():
164
+ """`ensure_unique` still resolves collisions when the generator draws
165
+ from a distribution instead of `random.sample` -- given a spread wide
166
+ enough to hold that many distinct values."""
167
+ tables = {
168
+ "t": TableDef(
169
+ name="t",
170
+ columns=[
171
+ ColumnDef(
172
+ "id",
173
+ "int",
174
+ {"pk"},
175
+ note={"distribution": "normal", "mean": 5000, "stddev": 500},
176
+ )
177
+ ],
178
+ )
179
+ }
180
+ data = generate_data_from_dbml(tables, [], base_rows=200, seed=6)
181
+ values = data["t"]["id"].tolist()
182
+ assert len(set(values)) == len(values)
183
+
184
+
185
+ # ---------------------------------------------------------
186
+ # Validation: every error names the column
187
+ # ---------------------------------------------------------
188
+ def test_distribution_on_a_non_numeric_column_is_rejected():
189
+ tables = _single_column_table(ColumnDef("label", "varchar", note={"distribution": "normal"}))
190
+ with pytest.raises(ValueError, match=r"t\.label"):
191
+ generate_data_from_dbml(tables, [], base_rows=5, seed=1)
192
+
193
+
194
+ def test_distribution_rejects_an_unknown_value():
195
+ tables = _single_column_table(ColumnDef("total", "numeric", note={"distribution": "gaussian"}))
196
+ with pytest.raises(ValueError, match=r"t\.total"):
197
+ generate_data_from_dbml(tables, [], base_rows=5, seed=1)
198
+
199
+
200
+ def test_stddev_without_distribution_is_rejected():
201
+ tables = _single_column_table(ColumnDef("total", "numeric", note={"stddev": 5}))
202
+ with pytest.raises(
203
+ ValueError, match=r't\.total: "stddev" only applies with "distribution": "normal"\.'
204
+ ):
205
+ generate_data_from_dbml(tables, [], base_rows=5, seed=1)
206
+
207
+
208
+ def test_stddev_with_the_wrong_distribution_is_rejected():
209
+ tables = _single_column_table(
210
+ ColumnDef("total", "numeric", note={"distribution": "exponential", "stddev": 5})
211
+ )
212
+ with pytest.raises(ValueError, match=r"t\.total"):
213
+ generate_data_from_dbml(tables, [], base_rows=5, seed=1)
214
+
215
+
216
+ def test_median_with_the_wrong_distribution_is_rejected():
217
+ tables = _single_column_table(
218
+ ColumnDef("total", "numeric", note={"distribution": "normal", "median": 5})
219
+ )
220
+ with pytest.raises(ValueError, match=r"t\.total"):
221
+ generate_data_from_dbml(tables, [], base_rows=5, seed=1)
222
+
223
+
224
+ def test_spread_with_the_wrong_distribution_is_rejected():
225
+ tables = _single_column_table(
226
+ ColumnDef("total", "numeric", note={"distribution": "normal", "spread": 0.5})
227
+ )
228
+ with pytest.raises(ValueError, match=r"t\.total"):
229
+ generate_data_from_dbml(tables, [], base_rows=5, seed=1)
230
+
231
+
232
+ def test_mean_without_a_matching_distribution_is_rejected():
233
+ tables = _single_column_table(ColumnDef("total", "numeric", note={"mean": 5}))
234
+ with pytest.raises(ValueError, match=r"t\.total"):
235
+ generate_data_from_dbml(tables, [], base_rows=5, seed=1)
236
+
237
+
238
+ @pytest.mark.parametrize(
239
+ "note",
240
+ [
241
+ {"distribution": "normal", "stddev": -1},
242
+ {"distribution": "normal", "stddev": 0},
243
+ {"distribution": "lognormal", "median": -5},
244
+ {"distribution": "lognormal", "spread": 0},
245
+ ],
246
+ )
247
+ def test_non_positive_stddev_median_spread_is_rejected(note):
248
+ tables = _single_column_table(ColumnDef("total", "numeric", note=note))
249
+ with pytest.raises(ValueError, match=r"t\.total"):
250
+ generate_data_from_dbml(tables, [], base_rows=5, seed=1)
251
+
252
+
253
+ def test_a_bool_value_is_rejected_where_a_number_is_expected():
254
+ tables = _single_column_table(
255
+ ColumnDef("total", "numeric", note={"distribution": "normal", "mean": True})
256
+ )
257
+ with pytest.raises(ValueError, match=r"t\.total"):
258
+ generate_data_from_dbml(tables, [], base_rows=5, seed=1)
@@ -0,0 +1,132 @@
1
+ """A lone `country` column reads as an international mix, not one repeated
2
+ country.
3
+
4
+ Since 1.3.0 every row draws one address from a per-table pool, and that
5
+ pool's country is always the locale's own -- right when a sibling
6
+ city/street/state/postcode column needs it to agree, wrong when `country` is
7
+ the only address-shaped column in the table. These tests pin: the mix (a
8
+ majority-but-not-all home-country share, several distinct countries), that a
9
+ sibling place column switches it back off, that a declared `country` type
10
+ behaves like the name, and that `distinct`/`null_rate`/determinism/locale
11
+ keep working the same way they do everywhere else.
12
+ """
13
+
14
+ from model2data.generate.core import generate_data_from_dbml
15
+ from model2data.parse.dbml import ColumnDef, TableDef
16
+
17
+
18
+ def _customers_table(*extra_columns: ColumnDef) -> TableDef:
19
+ return TableDef(
20
+ name="customers",
21
+ columns=[
22
+ ColumnDef(name="id", data_type="int", settings={"pk"}),
23
+ ColumnDef(name="country", data_type="varchar", settings={"not null"}),
24
+ *extra_columns,
25
+ ],
26
+ )
27
+
28
+
29
+ class TestLoneCountryColumn:
30
+ def test_lone_country_column_reads_as_an_international_mix(self):
31
+ df = generate_data_from_dbml(
32
+ tables={"customers": _customers_table()},
33
+ refs=[],
34
+ base_rows=500,
35
+ seed=1,
36
+ locale="nl_BE",
37
+ )["customers"]
38
+
39
+ counts = df["country"].value_counts(normalize=True)
40
+ assert len(counts) >= 5
41
+
42
+ home_share = counts.get("Belgium", 0.0)
43
+ assert 0.5 <= home_share <= 0.7
44
+ assert counts.idxmax() == "Belgium"
45
+
46
+ def test_a_sibling_place_column_keeps_the_country_single(self):
47
+ df = generate_data_from_dbml(
48
+ tables={
49
+ "customers": _customers_table(
50
+ ColumnDef(name="city", data_type="varchar", settings={"not null"})
51
+ )
52
+ },
53
+ refs=[],
54
+ base_rows=200,
55
+ seed=2,
56
+ locale="nl_BE",
57
+ )["customers"]
58
+
59
+ assert set(df["country"]) == {"Belgium"}
60
+
61
+ def test_declared_country_type_behaves_like_the_name(self):
62
+ df = generate_data_from_dbml(
63
+ tables={
64
+ "customers": TableDef(
65
+ name="customers",
66
+ columns=[
67
+ ColumnDef(name="id", data_type="int", settings={"pk"}),
68
+ # Named generically; the *declared type* is what
69
+ # says "country".
70
+ ColumnDef(name="hq", data_type="country", settings={"not null"}),
71
+ ],
72
+ )
73
+ },
74
+ refs=[],
75
+ base_rows=500,
76
+ seed=3,
77
+ locale="nl_BE",
78
+ )["customers"]
79
+
80
+ counts = df["hq"].value_counts(normalize=True)
81
+ assert len(counts) >= 5
82
+ assert 0.5 <= counts.get("Belgium", 0.0) <= 0.7
83
+
84
+ def test_distinct_hint_still_bounds_the_pool(self):
85
+ df = generate_data_from_dbml(
86
+ tables={
87
+ "customers": TableDef(
88
+ name="customers",
89
+ columns=[
90
+ ColumnDef(name="id", data_type="int", settings={"pk"}),
91
+ ColumnDef(
92
+ name="country",
93
+ data_type="varchar",
94
+ settings={"not null"},
95
+ note={"distinct": 3},
96
+ ),
97
+ ],
98
+ )
99
+ },
100
+ refs=[],
101
+ base_rows=300,
102
+ seed=4,
103
+ locale="nl_BE",
104
+ )["customers"]
105
+
106
+ assert df["country"].nunique() <= 3
107
+
108
+ def test_same_seed_reproduces_the_same_mix(self):
109
+ def run():
110
+ return generate_data_from_dbml(
111
+ tables={"customers": _customers_table()},
112
+ refs=[],
113
+ base_rows=200,
114
+ seed=5,
115
+ locale="nl_BE",
116
+ )["customers"]
117
+
118
+ first_run, second_run = run(), run()
119
+ assert list(first_run["country"]) == list(second_run["country"])
120
+
121
+ def test_default_locale_also_produces_a_mix(self):
122
+ df = generate_data_from_dbml(
123
+ tables={"customers": _customers_table()},
124
+ refs=[],
125
+ base_rows=500,
126
+ seed=6,
127
+ )["customers"]
128
+
129
+ counts = df["country"].value_counts(normalize=True)
130
+ assert len(counts) >= 5
131
+ assert 0.5 <= counts.get("United States", 0.0) <= 0.7
132
+ assert counts.idxmax() == "United States"
File without changes
File without changes
File without changes
File without changes
File without changes