model2data 1.0.0__tar.gz → 1.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. {model2data-1.0.0/model2data.egg-info → model2data-1.2.0}/PKG-INFO +1 -1
  2. {model2data-1.0.0 → model2data-1.2.0}/README.md +25 -12
  3. {model2data-1.0.0 → model2data-1.2.0}/model2data/cli.py +64 -0
  4. {model2data-1.0.0 → model2data-1.2.0}/model2data/generate/core.py +21 -5
  5. {model2data-1.0.0 → model2data-1.2.0}/model2data/generate/faker.py +34 -7
  6. {model2data-1.0.0 → model2data-1.2.0/model2data.egg-info}/PKG-INFO +1 -1
  7. {model2data-1.0.0 → model2data-1.2.0}/pyproject.toml +1 -1
  8. {model2data-1.0.0 → model2data-1.2.0}/tests/test_cli.py +55 -0
  9. {model2data-1.0.0 → model2data-1.2.0}/tests/test_faker_name_inference.py +67 -0
  10. {model2data-1.0.0 → model2data-1.2.0}/tests/test_generation.py +81 -8
  11. {model2data-1.0.0 → model2data-1.2.0}/tests/test_release_stress.py +8 -8
  12. {model2data-1.0.0 → model2data-1.2.0}/LICENSE +0 -0
  13. {model2data-1.0.0 → model2data-1.2.0}/README_PYPI.md +0 -0
  14. {model2data-1.0.0 → model2data-1.2.0}/model2data/__init__.py +0 -0
  15. {model2data-1.0.0 → model2data-1.2.0}/model2data/dbt/__init__.py +0 -0
  16. {model2data-1.0.0 → model2data-1.2.0}/model2data/dbt/project.py +0 -0
  17. {model2data-1.0.0 → model2data-1.2.0}/model2data/dbt/templates/dbt_project.yml.jinja +0 -0
  18. {model2data-1.0.0 → model2data-1.2.0}/model2data/dbt/templates/macros/generate_schema_name.sql +0 -0
  19. {model2data-1.0.0 → model2data-1.2.0}/model2data/dbt/templates/profiles.yml.jinja +0 -0
  20. {model2data-1.0.0 → model2data-1.2.0}/model2data/dbt/tests.py +0 -0
  21. {model2data-1.0.0 → model2data-1.2.0}/model2data/generate/__init__.py +0 -0
  22. {model2data-1.0.0 → model2data-1.2.0}/model2data/generate/relationships.py +0 -0
  23. {model2data-1.0.0 → model2data-1.2.0}/model2data/parse/__init__.py +0 -0
  24. {model2data-1.0.0 → model2data-1.2.0}/model2data/parse/dbml.py +0 -0
  25. {model2data-1.0.0 → model2data-1.2.0}/model2data/utils.py +0 -0
  26. {model2data-1.0.0 → model2data-1.2.0}/model2data.egg-info/SOURCES.txt +0 -0
  27. {model2data-1.0.0 → model2data-1.2.0}/model2data.egg-info/dependency_links.txt +0 -0
  28. {model2data-1.0.0 → model2data-1.2.0}/model2data.egg-info/entry_points.txt +0 -0
  29. {model2data-1.0.0 → model2data-1.2.0}/model2data.egg-info/requires.txt +0 -0
  30. {model2data-1.0.0 → model2data-1.2.0}/model2data.egg-info/top_level.txt +0 -0
  31. {model2data-1.0.0 → model2data-1.2.0}/setup.cfg +0 -0
  32. {model2data-1.0.0 → model2data-1.2.0}/tests/test_coverage_gaps.py +0 -0
  33. {model2data-1.0.0 → model2data-1.2.0}/tests/test_dbml_parser.py +0 -0
  34. {model2data-1.0.0 → model2data-1.2.0}/tests/test_dbml_parser_fuzz.py +0 -0
  35. {model2data-1.0.0 → model2data-1.2.0}/tests/test_dbt_integration.py +0 -0
  36. {model2data-1.0.0 → model2data-1.2.0}/tests/test_dbt_naming.py +0 -0
  37. {model2data-1.0.0 → model2data-1.2.0}/tests/test_dbt_project.py +0 -0
  38. {model2data-1.0.0 → model2data-1.2.0}/tests/test_dbt_tests.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: model2data
3
- Version: 1.0.0
3
+ Version: 1.2.0
4
4
  Summary: Generate analytics-ready datasets from DBML models
5
5
  Author: JB Analytica
6
6
  License-Expression: MIT
@@ -5,6 +5,8 @@
5
5
  [![codecov](https://codecov.io/gh/JB-Analytica/model2data/branch/main/graph/badge.svg)](https://codecov.io/gh/JB-Analytica/model2data)
6
6
  [![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
7
7
 
8
+ Built and maintained by [JB Analytica](https://www.jbanalytica.com/) — data platform architecture and analytics engineering.
9
+
8
10
  **Turn a data model into a running analytics stack in one command.**
9
11
 
10
12
  Give `model2data` a [DBML](https://dbml.dbdiagram.io/docs/) schema — hand-written or exported
@@ -13,17 +15,17 @@ from an existing database — and it generates realistic, relationship-preservin
13
15
  DuckDB or Postgres profile. No sample data to hunt down, no dbt boilerplate to hand-write, no
14
16
  production data to risk exposing.
15
17
 
18
+ A working analytics stack — real (synthetic) data, tested dbt models, queryable in
19
+ DuckDB — from a schema file, in seconds:
20
+
21
+ ![model2data generating a project and running it with dbt](https://raw.githubusercontent.com/JB-Analytica/model2data/main/assets/demo.gif)
22
+
16
23
  ```bash
17
24
  pip install model2data
18
25
  model2data --file examples/ecommerce.dbml --rows 200 --seed 42
19
26
  cd dbt_ecommerce && dbt build
20
27
  ```
21
28
 
22
- That's a working analytics stack — real (synthetic) data, tested dbt models, queryable in
23
- DuckDB — from a schema file, in seconds:
24
-
25
- ![model2data generating a project and running it with dbt](https://raw.githubusercontent.com/JB-Analytica/model2data/main/assets/demo.gif)
26
-
27
29
  ---
28
30
 
29
31
  ## Why this exists
@@ -37,7 +39,8 @@ access required.
37
39
  - **Privacy-safe.** Nothing but a schema definition goes in; nothing but synthetic data comes out.
38
40
  - **Realistic, not random.** Column names are matched against ~35 common patterns — `email`,
39
41
  `first_name`, `city`, `phone`, `company`, ... — so a column called `email` gets real-looking
40
- emails, not `Lorem ipsum` text.
42
+ emails, not `Lorem ipsum` text. Type a column with any Faker provider (`billing_country state`,
43
+ `sku ean13`) to pick its generator outright when the name is wrong for the data.
41
44
  - **Relationship-preserving.** Foreign keys resolve to real parent rows; tables are generated in
42
45
  dependency order.
43
46
  - **Deterministic.** Pass `--seed` and the same schema always produces the same data — safe to
@@ -95,8 +98,9 @@ flowchart LR
95
98
 
96
99
  1. **Parse.** Reads tables, columns, types, and `Ref` relationships from a DBML file.
97
100
  2. **Generate.** Produces synthetic values per column — typed generation for known SQL types
98
- (int, date, timestamp, ...), name-aware inference for everything else (`email`, `phone`,
99
- `city`, ...), foreign keys resolved against already-generated parent rows.
101
+ (int, date, timestamp, ...), then a Faker provider named as the type (`sku ean13`), then
102
+ name-aware inference for everything else (`email`, `phone`, `city`, ...), foreign keys
103
+ resolved against already-generated parent rows.
100
104
  3. **Scaffold.** Writes a complete dbt project around that data: CSV seeds, staging models that
101
105
  `ref()` those seeds, `not_null`/`unique`/`relationships` tests, `accepted_values` tests for
102
106
  DBML `Enum`-typed columns, singular SQL tests for composite primary/unique keys, table and
@@ -126,6 +130,14 @@ model2data --file examples/ecommerce.dbml --rows 200 --seed 42
126
130
 
127
131
  This creates a `dbt_ecommerce/` folder with your data and dbt setup.
128
132
 
133
+ Real schemas are rarely uniform. `--rows-for` sizes individual tables, so a handful of customers
134
+ can sit behind a large orders table the way they would in the warehouse you're modelling:
135
+
136
+ ```bash
137
+ model2data --file examples/ecommerce.dbml --rows 200 --seed 42 \
138
+ --rows-for customers=50 --rows-for order_items=5000
139
+ ```
140
+
129
141
  Run dbt to load, transform, and test the data:
130
142
 
131
143
  ```bash
@@ -251,10 +263,11 @@ If you're pinned to an older dbt-core, use model2data 0.5.x, which supported dow
251
263
 
252
264
  ## Project status
253
265
 
254
- As of `1.0.0`, model2data is considered **feature-complete for its intended use case**: turning a
255
- DBML schema into realistic synthetic data and a runnable dbt project, reliably. There's no active
256
- roadmap of new capabilities planned — the focus from here is maintenance: bug fixes, keeping pace
257
- with new dbt-core releases, and reviewing community contributions.
266
+ As of `1.0.0`, model2data is stable and feature-complete for its intended use case: turning a
267
+ DBML schema into realistic synthetic data and a runnable dbt project, reliably. It is actively
268
+ maintained — CI runs a real `dbt build` against both the oldest supported dbt-core and the newest
269
+ release on every push, so compatibility is proven rather than assumed. New capabilities are not
270
+ the focus; correctness, dbt-core compatibility and community contributions are.
258
271
 
259
272
  Ideas that came up during development but were deliberately left out of scope, in case anyone
260
273
  wants to pick them up as a contribution:
@@ -27,6 +27,53 @@ from model2data.utils import normalize_identifier
27
27
 
28
28
  SUPPORTED_ADAPTERS = ("duckdb", "postgres")
29
29
 
30
+
31
+ def _parse_row_overrides(
32
+ raw: Optional[list[str]],
33
+ tables: dict,
34
+ ) -> dict[str, int]:
35
+ """Turn repeated `--rows-for TABLE=N` values into a {table: rows} mapping.
36
+
37
+ Fails loudly rather than silently ignoring a typo: naming a table that
38
+ isn't in the schema almost always means a misspelling, and quietly
39
+ generating the default row count for it would be discovered only by
40
+ counting rows in the output.
41
+ """
42
+ # `main` is also called directly as a plain function (see tests/), which
43
+ # bypasses Typer and leaves this parameter holding its `OptionInfo` default
44
+ # rather than None. Anything that isn't an actual list means "not supplied".
45
+ if not isinstance(raw, (list, tuple)):
46
+ return {}
47
+
48
+ overrides: dict[str, int] = {}
49
+ for item in raw:
50
+ table_name, separator, count = item.partition("=")
51
+ table_name = table_name.strip()
52
+ if not separator or not table_name:
53
+ raise typer.BadParameter(f"Expected TABLE=N, got {item!r}.", param_hint="--rows-for")
54
+
55
+ try:
56
+ rows = int(count)
57
+ except ValueError:
58
+ raise typer.BadParameter(
59
+ f"Row count for {table_name!r} must be a whole number, got {count!r}.",
60
+ param_hint="--rows-for",
61
+ ) from None
62
+ if rows < 1:
63
+ raise typer.BadParameter(
64
+ f"Row count for {table_name!r} must be at least 1, got {rows}.",
65
+ param_hint="--rows-for",
66
+ )
67
+ if table_name not in tables:
68
+ known = ", ".join(sorted(tables)) or "none"
69
+ raise typer.BadParameter(
70
+ f"No table named {table_name!r} in this schema. Tables: {known}.",
71
+ param_hint="--rows-for",
72
+ )
73
+ overrides[table_name] = rows
74
+ return overrides
75
+
76
+
30
77
  app = typer.Typer(
31
78
  help=(
32
79
  "model2data: Generate analytics-ready datasets from DBML models.\n\n"
@@ -59,6 +106,17 @@ def main(
59
106
  min=10,
60
107
  help="Number of rows to generate per table.",
61
108
  ),
109
+ # noqa: B008 is only needed here (not on the other options) because a
110
+ # repeatable option must be annotated with a mutable `list` type.
111
+ rows_for: Optional[list[str]] = typer.Option( # noqa: B008
112
+ None,
113
+ "--rows-for",
114
+ metavar="TABLE=N",
115
+ help=(
116
+ "Row count for one table, overriding --rows. Repeatable, e.g.\n"
117
+ "--rows-for customers=200 --rows-for orders=5000."
118
+ ),
119
+ ),
62
120
  seed: Optional[int] = typer.Option(
63
121
  None,
64
122
  "--seed",
@@ -124,6 +182,11 @@ def main(
124
182
  typer.echo("❌ No tables found in the provided DBML file.")
125
183
  raise typer.Exit(1)
126
184
 
185
+ # Validated before anything touches the filesystem: a typo'd table name here
186
+ # should not leave a half-scaffolded project behind for the next run to trip
187
+ # over with a confusing "destination already exists".
188
+ row_overrides = _parse_row_overrides(rows_for, tables)
189
+
127
190
  project_name = normalize_identifier(name or file.stem)
128
191
  dest = Path.cwd() / f"dbt_{project_name}"
129
192
  profile_name = f"{project_name}_profile"
@@ -150,6 +213,7 @@ def main(
150
213
  refs=refs,
151
214
  base_rows=rows,
152
215
  seed=seed,
216
+ row_overrides=row_overrides,
153
217
  )
154
218
 
155
219
  # -------------------------
@@ -2,6 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  import random
4
4
  from collections import defaultdict, deque
5
+ from collections.abc import Mapping
5
6
  from typing import Optional
6
7
 
7
8
  import pandas as pd
@@ -64,10 +65,19 @@ def generate_data_from_dbml(
64
65
  refs: list[dict],
65
66
  base_rows: int = 100,
66
67
  seed: Optional[int] = None,
68
+ row_overrides: Optional[Mapping[str, int]] = None,
67
69
  ) -> dict[str, pd.DataFrame]:
68
70
  """
69
71
  Generate synthetic datasets from parsed DBML definitions.
70
72
 
73
+ `base_rows` is the row count for every table; `row_overrides` sets it per
74
+ table, keyed by DBML table name. Real schemas are rarely uniform -- a
75
+ handful of dimension rows against a fact table two orders of magnitude
76
+ larger is the normal shape, and generating 100 of each makes joins and
77
+ aggregates behave nothing like the warehouse being modelled. Names not
78
+ present in `row_overrides` fall back to `base_rows`; unknown names are
79
+ ignored.
80
+
71
81
  This function is deterministic if a seed is provided.
72
82
  It performs no filesystem I/O and returns pandas DataFrames.
73
83
  """
@@ -93,7 +103,7 @@ def generate_data_from_dbml(
93
103
 
94
104
  for table_name in ordered_tables:
95
105
  table_def = tables[table_name]
96
- row_count = _determine_row_count(table_def.name, base_rows)
106
+ row_count = _determine_row_count(table_def.name, base_rows, row_overrides)
97
107
 
98
108
  data: dict[str, list] = {}
99
109
 
@@ -341,10 +351,16 @@ def _resolve_self_referencing_fks(
341
351
  return df
342
352
 
343
353
 
344
- def _determine_row_count(table_name: str, base_rows: int) -> int:
345
- """
346
- Return the base number of rows for all tables.
347
- """
354
+ def _determine_row_count(
355
+ table_name: str,
356
+ base_rows: int,
357
+ row_overrides: Optional[Mapping[str, int]] = None,
358
+ ) -> int:
359
+ """Return the row count for one table: its override, else `base_rows`."""
360
+ if row_overrides:
361
+ override = row_overrides.get(table_name)
362
+ if override is not None:
363
+ return override
348
364
  return base_rows
349
365
 
350
366
 
@@ -150,6 +150,33 @@ def _infer_by_name(column_name: str) -> Optional[Callable[[], object]]:
150
150
  return None
151
151
 
152
152
 
153
+ # Type names that are ordinary SQL types first and Faker providers only by
154
+ # coincidence. Someone typing `email text` means the SQL type, so these must
155
+ # not count as a deliberate choice of provider -- otherwise the commonest
156
+ # declaration in any schema would stop generating emails.
157
+ _SQL_TYPES_SHADOWING_A_PROVIDER = frozenset({"text", "json", "jsonb", "xml", "binary", "year"})
158
+
159
+
160
+ def _infer_by_type(base_type: str) -> Optional[Callable[[], object]]:
161
+ """The provider a column's declared type names, if it names one deliberately.
162
+
163
+ `sku ean13` and `home_state state` are the user saying which generator they
164
+ want, in the only place DBML gives them to say it. That has to outrank the
165
+ guess made from the column's name: a name pattern is inferred, a type is
166
+ declared, and `first_name email` silently generating first names -- the
167
+ type having no effect whatsoever -- is the single most confusing thing this
168
+ module did.
169
+ """
170
+ if base_type in _SQL_TYPES_SHADOWING_A_PROVIDER:
171
+ return None
172
+ try:
173
+ fake.format(base_type)
174
+ except (AttributeError, TypeError):
175
+ # Not a provider, or one that needs arguments: nothing was declared.
176
+ return None
177
+ return lambda: fake.format(base_type)
178
+
179
+
153
180
  # ---------------------------------------------------------
154
181
  # Public API
155
182
  # ---------------------------------------------------------
@@ -278,16 +305,16 @@ def generate_column_values(
278
305
  values = [_random_datetime().isoformat(sep=" ") for _ in range(row_count)]
279
306
 
280
307
  # -----------------------------------------------------
281
- # Untyped / generic string columns: infer intent from the
282
- # column name first (email, city, phone...), then fall back
283
- # to a literal Faker provider name, then to a generic value.
308
+ # Untyped / generic string columns: honour a type that names
309
+ # a Faker provider (`sku ean13`), then infer intent from the
310
+ # column name (email, city, phone...), then a generic value.
284
311
  # -----------------------------------------------------
285
312
  else:
286
- name_generator = _infer_by_name(column.name)
287
- if name_generator is not None:
288
- values = [name_generator() for _ in range(row_count)]
313
+ generator = _infer_by_type(base_type) or _infer_by_name(column.name)
314
+ if generator is not None:
315
+ values = [generator() for _ in range(row_count)]
289
316
  values = (
290
- _deduplicate(values, name_generator, column_name=unique_label)
317
+ _deduplicate(values, generator, column_name=unique_label)
291
318
  if ensure_unique
292
319
  else values
293
320
  )
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: model2data
3
- Version: 1.0.0
3
+ Version: 1.2.0
4
4
  Summary: Generate analytics-ready datasets from DBML models
5
5
  Author: JB Analytica
6
6
  License-Expression: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "model2data"
7
- version = "1.0.0"
7
+ version = "1.2.0"
8
8
  description = "Generate analytics-ready datasets from DBML models"
9
9
  readme = "README_PYPI.md"
10
10
  requires-python = ">=3.10"
@@ -777,3 +777,58 @@ def test_cli_summary_is_quiet_when_uniqueness_is_satisfiable(tmp_path):
777
777
  assert result.exit_code == 0
778
778
  assert "duplicate row(s)" not in result.stdout
779
779
  assert "duplicate value(s)" not in result.stdout
780
+
781
+
782
+ ROWS_FOR_SCHEMA = """
783
+ Table users {
784
+ id int [pk]
785
+ email email [unique]
786
+ }
787
+ Table orders {
788
+ id int [pk]
789
+ user_id int [not null]
790
+ }
791
+ Ref: orders.user_id > users.id
792
+ """
793
+
794
+
795
+ def _run_rows_for(tmp_path, *extra_args):
796
+ dbml_file = tmp_path / "shop.dbml"
797
+ dbml_file.write_text(ROWS_FOR_SCHEMA)
798
+ cwd = os.getcwd()
799
+ os.chdir(tmp_path)
800
+ try:
801
+ return runner.invoke(app, ["--file", str(dbml_file), "--rows", "50", *extra_args])
802
+ finally:
803
+ os.chdir(cwd)
804
+
805
+
806
+ def test_cli_rows_for_sets_row_counts_per_table(tmp_path):
807
+ result = _run_rows_for(tmp_path, "--rows-for", "orders=120")
808
+ assert result.exit_code == 0, result.output
809
+
810
+ seeds = tmp_path / "dbt_shop" / "seeds" / "raw"
811
+ assert len(pd.read_csv(seeds / "orders.csv")) == 120
812
+ assert len(pd.read_csv(seeds / "users.csv")) == 50
813
+
814
+
815
+ def test_cli_rows_for_is_repeatable(tmp_path):
816
+ result = _run_rows_for(tmp_path, "--rows-for", "users=10", "--rows-for", "orders=200")
817
+ assert result.exit_code == 0, result.output
818
+
819
+ seeds = tmp_path / "dbt_shop" / "seeds" / "raw"
820
+ assert len(pd.read_csv(seeds / "users.csv")) == 10
821
+ assert len(pd.read_csv(seeds / "orders.csv")) == 200
822
+
823
+
824
+ def test_cli_rows_for_rejects_an_unknown_table(tmp_path):
825
+ """A typo'd table name would otherwise be discovered by counting rows."""
826
+ result = _run_rows_for(tmp_path, "--rows-for", "userz=10")
827
+ assert result.exit_code != 0
828
+ assert "No table named 'userz'" in result.output
829
+
830
+
831
+ def test_cli_rows_for_rejects_malformed_values(tmp_path):
832
+ assert "Expected TABLE=N" in _run_rows_for(tmp_path, "--rows-for", "users").output
833
+ assert "whole number" in _run_rows_for(tmp_path, "--rows-for", "users=lots").output
834
+ assert "at least 1" in _run_rows_for(tmp_path, "--rows-for", "users=0").output
@@ -82,6 +82,73 @@ class TestNameInference:
82
82
  assert result == ["x", "x", "x"]
83
83
 
84
84
 
85
+ class TestTypeBeatsName:
86
+ """A declared Faker provider type outranks the guess made from the name.
87
+
88
+ DBML has one place to say which generator a column should use -- its type --
89
+ and until this held, a recognised column name silently overruled it:
90
+ `home_state state` generated countries and `first_name email` generated
91
+ first names, with the declared type having no effect at all.
92
+ """
93
+
94
+ def test_a_declared_provider_overrules_the_name_pattern(self):
95
+ values = generate_column_values(
96
+ ColumnDef(name="first_name", data_type="email", settings={"not null"}),
97
+ row_count=20,
98
+ )
99
+ assert all(EMAIL_RE.match(v) for v in values)
100
+
101
+ def test_a_column_can_be_typed_against_its_own_name(self):
102
+ # The case the feature exists for: a state column that isn't called one.
103
+ # Without this the `country` in its name won and it generated countries.
104
+ from faker.providers.address.en_US import Provider as UsAddress
105
+
106
+ values = generate_column_values(
107
+ ColumnDef(name="billing_country", data_type="state", settings={"not null"}),
108
+ row_count=40,
109
+ )
110
+ assert set(values) <= set(UsAddress.states)
111
+
112
+ def test_a_plain_sql_type_does_not_count_as_a_declaration(self):
113
+ # `text` and `json` are Faker providers by coincidence; a column typed
114
+ # that way means the SQL type, and its name must still be inferred.
115
+ for sql_type in ("text", "json", "varchar", "char(64)"):
116
+ values = generate_column_values(
117
+ ColumnDef(name="email", data_type=sql_type, settings={"not null"}),
118
+ row_count=10,
119
+ )
120
+ assert all(EMAIL_RE.match(v) for v in values), sql_type
121
+
122
+ def test_an_unknown_type_still_falls_back_to_the_name(self):
123
+ reset_stats()
124
+ values = generate_column_values(
125
+ ColumnDef(name="city", data_type="weird_custom_type", settings={"not null"}),
126
+ row_count=10,
127
+ )
128
+ assert all(isinstance(value, str) and value for value in values)
129
+ assert get_unmapped_columns() == []
130
+
131
+ def test_a_declared_provider_is_still_deduplicated_when_unique(self):
132
+ values = generate_column_values(
133
+ ColumnDef(name="first_name", data_type="email", settings={"not null", "unique"}),
134
+ row_count=50,
135
+ ensure_unique=True,
136
+ )
137
+ assert len(values) == len(set(values)) == 50
138
+
139
+ def test_structured_types_are_untouched_by_this(self):
140
+ # `date` and `boolean` are providers too, but they are decided long
141
+ # before either inference runs, and must keep their real Python types.
142
+ dates = generate_column_values(
143
+ ColumnDef(name="signup_date", data_type="date", settings={"not null"}), row_count=5
144
+ )
145
+ assert all(hasattr(value, "year") for value in dates)
146
+ flags = generate_column_values(
147
+ ColumnDef(name="is_active", data_type="boolean", settings={"not null"}), row_count=5
148
+ )
149
+ assert set(flags) <= {True, False}
150
+
151
+
85
152
  class TestEnumGeneration:
86
153
  def test_enum_values_only_ever_come_from_the_enum_set(self):
87
154
  allowed = {"active", "inactive", "pending"}
@@ -670,7 +670,7 @@ def test_enum_column_whose_name_contains_int_does_not_crash_dtype_coercion():
670
670
  assert set(df["maintenance_type"]) <= {"oil_change", "repair", "cleaning"}
671
671
 
672
672
 
673
- def test_composite_key_dedup_retry_preserves_fk_validity(monkeypatch):
673
+ def test_composite_key_dedup_retry_preserves_fk_validity():
674
674
  # Regression test: a join/bridge table's composite key is almost always
675
675
  # built from FK columns (posts/tags -> post_tags is the canonical
676
676
  # example). When two parent tables are small relative to the bridge
@@ -679,7 +679,6 @@ def test_composite_key_dedup_retry_preserves_fk_validity(monkeypatch):
679
679
  # type-based generator instead of resampling from the real parent id
680
680
  # pool, silently producing post_id/tag_id values that referenced no
681
681
  # real parent row at all.
682
- import model2data.generate.core as core
683
682
 
684
683
  tables = {
685
684
  "posts": TableDef(name="posts", columns=[ColumnDef("id", "int", {"pk"})]),
@@ -710,12 +709,13 @@ def test_composite_key_dedup_retry_preserves_fk_validity(monkeypatch):
710
709
 
711
710
  # Force posts/tags to 5 rows each (25 possible combos) while post_tags
712
711
  # gets 200 rows, guaranteeing heavy dedup-retry activity.
713
- def small_parents_large_bridge(table_name, base_rows):
714
- return 5 if table_name in ("posts", "tags") else base_rows
715
-
716
- monkeypatch.setattr(core, "_determine_row_count", small_parents_large_bridge)
717
-
718
- data = generate_data_from_dbml(tables, refs, base_rows=200, seed=123)
712
+ data = generate_data_from_dbml(
713
+ tables,
714
+ refs,
715
+ base_rows=200,
716
+ seed=123,
717
+ row_overrides={"posts": 5, "tags": 5},
718
+ )
719
719
 
720
720
  post_ids = set(data["posts"]["id"].tolist())
721
721
  tag_ids = set(data["tags"]["id"].tolist())
@@ -885,3 +885,76 @@ def test_timestamp_generation_is_reproducible_across_runs():
885
885
  assert list(first["created_at"]) == list(second["created_at"])
886
886
  # No sub-second component at all, so a CSV round-trip stays stable too.
887
887
  assert all(ts.endswith(":00") or "." not in ts for ts in map(str, first["created_at"]))
888
+
889
+
890
+ def _users_and_orders() -> tuple[dict, list[dict]]:
891
+ tables = {
892
+ "users": TableDef(
893
+ name="users",
894
+ columns=[ColumnDef("id", "int", {"pk"}), ColumnDef("email", "email", {"unique"})],
895
+ ),
896
+ "orders": TableDef(
897
+ name="orders",
898
+ columns=[
899
+ ColumnDef("id", "int", {"pk"}),
900
+ ColumnDef("user_id", "int", {"not null"}),
901
+ ColumnDef("total", "numeric"),
902
+ ],
903
+ ),
904
+ }
905
+ refs = [
906
+ {
907
+ "source_table": "orders",
908
+ "source_column": "user_id",
909
+ "target_table": "users",
910
+ "target_column": "id",
911
+ }
912
+ ]
913
+ return tables, refs
914
+
915
+
916
+ def test_row_overrides_set_row_counts_per_table():
917
+ tables, refs = _users_and_orders()
918
+
919
+ data = generate_data_from_dbml(
920
+ tables, refs, base_rows=50, seed=1, row_overrides={"orders": 300}
921
+ )
922
+
923
+ assert len(data["orders"]) == 300
924
+ assert len(data["users"]) == 50, "tables without an override keep base_rows"
925
+
926
+
927
+ def test_row_overrides_default_to_base_rows_and_ignore_unknown_tables():
928
+ tables, refs = _users_and_orders()
929
+
930
+ data = generate_data_from_dbml(
931
+ tables, refs, base_rows=20, seed=1, row_overrides={"nonexistent": 999}
932
+ )
933
+
934
+ assert {name: len(df) for name, df in data.items()} == {"users": 20, "orders": 20}
935
+
936
+
937
+ def test_foreign_keys_stay_valid_when_the_child_is_much_larger_than_its_parent():
938
+ """The point of per-table counts: a small dimension against a large fact."""
939
+ tables, refs = _users_and_orders()
940
+
941
+ data = generate_data_from_dbml(
942
+ tables, refs, base_rows=100, seed=42, row_overrides={"users": 5, "orders": 2000}
943
+ )
944
+
945
+ assert len(data["users"]) == 5
946
+ assert len(data["orders"]) == 2000
947
+ assert set(data["orders"]["user_id"]) <= set(data["users"]["id"])
948
+ # A 5-row parent still has to have unique keys for the FK to mean anything.
949
+ assert data["users"]["id"].is_unique
950
+
951
+
952
+ def test_row_overrides_are_deterministic_with_a_seed():
953
+ tables, refs = _users_and_orders()
954
+ overrides = {"users": 25, "orders": 175}
955
+
956
+ first = generate_data_from_dbml(tables, refs, base_rows=50, seed=9, row_overrides=overrides)
957
+ second = generate_data_from_dbml(tables, refs, base_rows=50, seed=9, row_overrides=overrides)
958
+
959
+ assert first["users"].equals(second["users"])
960
+ assert first["orders"].equals(second["orders"])
@@ -26,7 +26,6 @@ from pathlib import Path
26
26
 
27
27
  import pytest
28
28
 
29
- import model2data.generate.core as core
30
29
  from model2data.generate.core import generate_data_from_dbml
31
30
  from model2data.generate.relationships import build_fk_lookup, classify_refs
32
31
  from model2data.parse.dbml import ColumnDef, TableDef, parse_dbml
@@ -187,7 +186,7 @@ def _bridge_schema():
187
186
  )
188
187
  @pytest.mark.parametrize("seed", list(range(1, 16))) # 15 seeds
189
188
  def test_composite_key_dedup_extreme_value_space_stress(
190
- seed, parent_rows, bridge_rows, uniqueness_achievable, monkeypatch
189
+ seed, parent_rows, bridge_rows, uniqueness_achievable
191
190
  ):
192
191
  # Extends the original bug-1 regression test
193
192
  # (test_composite_key_dedup_retry_preserves_fk_validity in
@@ -199,12 +198,13 @@ def test_composite_key_dedup_extreme_value_space_stress(
199
198
  # surface at all.
200
199
  tables, refs = _bridge_schema()
201
200
 
202
- def sized_row_counts(table_name, base_rows):
203
- return parent_rows if table_name in ("posts", "tags") else base_rows
204
-
205
- monkeypatch.setattr(core, "_determine_row_count", sized_row_counts)
206
-
207
- data = generate_data_from_dbml(tables, refs, base_rows=bridge_rows, seed=seed)
201
+ data = generate_data_from_dbml(
202
+ tables,
203
+ refs,
204
+ base_rows=bridge_rows,
205
+ seed=seed,
206
+ row_overrides={"posts": parent_rows, "tags": parent_rows},
207
+ )
208
208
 
209
209
  post_ids = set(data["posts"]["id"].tolist())
210
210
  tag_ids = set(data["tags"]["id"].tolist())
File without changes
File without changes
File without changes