model2data 0.3.0__tar.gz → 0.4.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. model2data-0.4.3/PKG-INFO +285 -0
  2. model2data-0.4.3/README.md +260 -0
  3. {model2data-0.3.0 → model2data-0.4.3}/model2data/cli.py +13 -1
  4. {model2data-0.3.0 → model2data-0.4.3}/model2data/dbt/project.py +1 -1
  5. model2data-0.4.3/model2data/dbt/tests.py +276 -0
  6. {model2data-0.3.0 → model2data-0.4.3}/model2data/generate/core.py +101 -0
  7. {model2data-0.3.0 → model2data-0.4.3}/model2data/generate/faker.py +69 -3
  8. model2data-0.4.3/model2data/parse/dbml.py +452 -0
  9. model2data-0.4.3/model2data.egg-info/PKG-INFO +285 -0
  10. {model2data-0.3.0 → model2data-0.4.3}/model2data.egg-info/SOURCES.txt +1 -0
  11. {model2data-0.3.0 → model2data-0.4.3}/model2data.egg-info/requires.txt +0 -1
  12. {model2data-0.3.0 → model2data-0.4.3}/pyproject.toml +2 -8
  13. {model2data-0.3.0 → model2data-0.4.3}/tests/test_cli.py +65 -0
  14. {model2data-0.3.0 → model2data-0.4.3}/tests/test_dbml_parser.py +540 -7
  15. model2data-0.4.3/tests/test_dbt_integration.py +104 -0
  16. {model2data-0.3.0 → model2data-0.4.3}/tests/test_dbt_project.py +1 -1
  17. model2data-0.4.3/tests/test_dbt_tests.py +348 -0
  18. model2data-0.4.3/tests/test_faker_name_inference.py +191 -0
  19. {model2data-0.3.0 → model2data-0.4.3}/tests/test_generation.py +134 -0
  20. model2data-0.3.0/PKG-INFO +0 -177
  21. model2data-0.3.0/README.md +0 -151
  22. model2data-0.3.0/model2data/dbt/tests.py +0 -92
  23. model2data-0.3.0/model2data/parse/dbml.py +0 -227
  24. model2data-0.3.0/model2data.egg-info/PKG-INFO +0 -177
  25. model2data-0.3.0/tests/test_dbt_tests.py +0 -64
  26. model2data-0.3.0/tests/test_faker_name_inference.py +0 -79
  27. {model2data-0.3.0 → model2data-0.4.3}/LICENSE +0 -0
  28. {model2data-0.3.0 → model2data-0.4.3}/model2data/__init__.py +0 -0
  29. {model2data-0.3.0 → model2data-0.4.3}/model2data/dbt/__init__.py +0 -0
  30. {model2data-0.3.0 → model2data-0.4.3}/model2data/dbt/templates/dbt_project.yml.jinja +0 -0
  31. {model2data-0.3.0 → model2data-0.4.3}/model2data/dbt/templates/macros/generate_schema_name.sql +0 -0
  32. {model2data-0.3.0 → model2data-0.4.3}/model2data/dbt/templates/profiles.yml.jinja +0 -0
  33. {model2data-0.3.0 → model2data-0.4.3}/model2data/generate/__init__.py +0 -0
  34. {model2data-0.3.0 → model2data-0.4.3}/model2data/generate/relationships.py +0 -0
  35. {model2data-0.3.0 → model2data-0.4.3}/model2data/parse/__init__.py +0 -0
  36. {model2data-0.3.0 → model2data-0.4.3}/model2data/utils.py +0 -0
  37. {model2data-0.3.0 → model2data-0.4.3}/model2data.egg-info/dependency_links.txt +0 -0
  38. {model2data-0.3.0 → model2data-0.4.3}/model2data.egg-info/entry_points.txt +0 -0
  39. {model2data-0.3.0 → model2data-0.4.3}/model2data.egg-info/top_level.txt +0 -0
  40. {model2data-0.3.0 → model2data-0.4.3}/setup.cfg +0 -0
  41. {model2data-0.3.0 → model2data-0.4.3}/tests/test_coverage_gaps.py +0 -0
  42. {model2data-0.3.0 → model2data-0.4.3}/tests/test_dbt_naming.py +0 -0
@@ -0,0 +1,285 @@
1
+ Metadata-Version: 2.4
2
+ Name: model2data
3
+ Version: 0.4.3
4
+ Summary: Generate analytics-ready datasets from DBML models
5
+ Requires-Python: >=3.10
6
+ Description-Content-Type: text/markdown
7
+ License-File: LICENSE
8
+ Requires-Dist: dbt-core>=1.5.0
9
+ Requires-Dist: dbt-duckdb>=1.5.0
10
+ Requires-Dist: faker>=37.12.0
11
+ Requires-Dist: pandas>=2.3.3
12
+ Requires-Dist: pyyaml>=6.0.3
13
+ Requires-Dist: typer>=0.20.0
14
+ Provides-Extra: postgres
15
+ Requires-Dist: dbt-postgres>=1.5.0; extra == "postgres"
16
+ Provides-Extra: dev
17
+ Requires-Dist: pytest; extra == "dev"
18
+ Requires-Dist: pytest-cov; extra == "dev"
19
+ Requires-Dist: pre-commit; extra == "dev"
20
+ Requires-Dist: ruff>=0.1.0; extra == "dev"
21
+ Requires-Dist: ty>=0.0.4; extra == "dev"
22
+ Requires-Dist: types-pyyaml; extra == "dev"
23
+ Requires-Dist: poethepoet>=0.38.0; extra == "dev"
24
+ Dynamic: license-file
25
+
26
+ # model2data
27
+
28
+ [![PyPI](https://img.shields.io/pypi/v/model2data)](https://pypi.org/project/model2data/)
29
+ [![CI](https://github.com/JB-Analytica/model2data/actions/workflows/ci.yml/badge.svg)](https://github.com/JB-Analytica/model2data/actions/workflows/ci.yml)
30
+ [![codecov](https://codecov.io/gh/JB-Analytica/model2data/branch/main/graph/badge.svg)](https://codecov.io/gh/JB-Analytica/model2data)
31
+ [![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
32
+
33
+ **Turn a data model into a running analytics stack in one command.**
34
+
35
+ Give `model2data` a [DBML](https://dbml.dbdiagram.io/docs/) schema — hand-written or exported
36
+ from an existing database — and it generates realistic, relationship-preserving synthetic data
37
+ *and* a complete, runnable dbt project around it: seeds, staging models, sources, tests, and a
38
+ DuckDB or Postgres profile. No sample data to hunt down, no dbt boilerplate to hand-write, no
39
+ production data to risk exposing.
40
+
41
+ ```bash
42
+ pip install model2data
43
+ model2data --file examples/hackernews.dbml --rows 200 --seed 42
44
+ cd dbt_hackernews && dbt seed && dbt run
45
+ ```
46
+
47
+ That's a working analytics stack — real (synthetic) data, tested dbt models, queryable in
48
+ DuckDB — from a schema file, in seconds:
49
+
50
+ ![model2data generating a project and running it with dbt](assets/demo.gif)
51
+
52
+ ---
53
+
54
+ ## Why this exists
55
+
56
+ Analytics engineers hit the same wall constantly: you need realistic data to build or test a
57
+ pipeline, but production data is off-limits (privacy, access, scale), and hand-rolling mock CSVs
58
+ is tedious and doesn't scale past two tables. `model2data` closes that gap — from a schema
59
+ definition to a seeded, tested dbt project you can actually run, with no database or production
60
+ access required.
61
+
62
+ - **Privacy-safe.** Nothing but a schema definition goes in; nothing but synthetic data comes out.
63
+ - **Realistic, not random.** Column names are matched against ~35 common patterns — `email`,
64
+ `first_name`, `city`, `phone`, `company`, ... — so a column called `email` gets real-looking
65
+ emails, not `Lorem ipsum` text.
66
+ - **Relationship-preserving.** Foreign keys resolve to real parent rows; tables are generated in
67
+ dependency order.
68
+ - **Deterministic.** Pass `--seed` and the same schema always produces the same data — safe to
69
+ commit fixtures, safe to diff across CI runs.
70
+ - **A real dbt project, not just CSVs.** Seeds, staging models, sources, schema tests, and a
71
+ ready-to-use profile — the thing you'd otherwise spend an afternoon scaffolding by hand.
72
+
73
+ ## Who is model2data for?
74
+
75
+ - **Analytics engineers** — generate realistic datasets and a working dbt project without
76
+ waiting on production access.
77
+ - **Data engineers** — produce deterministic test data from an existing schema for pipeline and
78
+ migration testing.
79
+ - **Software & data teams** — prototype integrations and analytics workflows without exposing
80
+ production data.
81
+ - **Consultants & architects** — spin up realistic environments for demos, workshops, and
82
+ architecture validation in minutes, not hours.
83
+
84
+ ## How it works
85
+
86
+ ```mermaid
87
+ flowchart LR
88
+ subgraph input [" "]
89
+ A["📄 DBML schema"]
90
+ end
91
+
92
+ subgraph m2d ["model2data"]
93
+ direction LR
94
+ B["Parse\ntables, columns,\nrelationships"] --> C["Generate\nFaker + name-aware\ninference, FK-aware"]
95
+ C --> D["Scaffold\nseeds · staging models\ntests · profile"]
96
+ end
97
+
98
+ subgraph output ["Generated dbt project"]
99
+ direction TB
100
+ E["seeds/*.csv"]
101
+ F["models/staging/*.sql + *.yml"]
102
+ G["profiles.yml\n(DuckDB or Postgres)"]
103
+ end
104
+
105
+ A --> B
106
+ D --> E
107
+ D --> F
108
+ D --> G
109
+ E & F & G --> H["dbt seed && dbt run"]
110
+ H --> I[("Analytics-ready\ndataset")]
111
+
112
+ classDef m2dStyle fill:#0A3866,stroke:#2196F0,color:#F6F8FB
113
+ classDef outStyle fill:#182333,stroke:#A8C9EE,color:#F6F8FB
114
+ classDef endStyle fill:#FA9306,stroke:#FA9306,color:#182333
115
+ class B,C,D m2dStyle
116
+ class E,F,G outStyle
117
+ class H,I endStyle
118
+ ```
119
+
120
+ 1. **Parse.** Reads tables, columns, types, and `Ref` relationships from a DBML file.
121
+ 2. **Generate.** Produces synthetic values per column — typed generation for known SQL types
122
+ (int, date, timestamp, ...), name-aware inference for everything else (`email`, `phone`,
123
+ `city`, ...), foreign keys resolved against already-generated parent rows.
124
+ 3. **Scaffold.** Writes a complete dbt project around that data: CSV seeds, staging models with
125
+ `source`/`not_null`/`unique`/`relationships` tests, `accepted_values` tests for DBML
126
+ `Enum`-typed columns, singular SQL tests for composite primary/unique keys, table and column
127
+ `description:` fields pulled from DBML notes, and a profile for DuckDB (zero-config,
128
+ file-based) or Postgres.
129
+
130
+ ---
131
+
132
+ ## Installation
133
+
134
+ ```bash
135
+ pip install model2data
136
+ ```
137
+
138
+ ---
139
+
140
+ ## Quick start
141
+
142
+ We provide an example Hacker News dataset in `examples/hackernews.dbml`.
143
+
144
+ Generate a project with synthetic data:
145
+
146
+ ```bash
147
+ model2data --file examples/hackernews.dbml --rows 200 --seed 42
148
+ ```
149
+
150
+ This creates a `dbt_hackernews/` folder with your data and dbt setup.
151
+
152
+ Run dbt to load and transform the data:
153
+
154
+ ```bash
155
+ cd dbt_hackernews
156
+ dbt deps
157
+ dbt seed
158
+ dbt run
159
+ ```
160
+
161
+ Your analytics-ready dataset is now in DuckDB!
162
+
163
+ To target Postgres instead, install the extra and pass `--adapter postgres`:
164
+
165
+ ```bash
166
+ pip install "model2data[postgres]"
167
+ model2data --file examples/hackernews.dbml --rows 200 --seed 42 --adapter postgres
168
+ ```
169
+
170
+ Connection details are read from environment variables (`MODEL2DATA_PG_HOST`, `MODEL2DATA_PG_PORT`, `MODEL2DATA_PG_USER`, `MODEL2DATA_PG_PASSWORD`, `MODEL2DATA_PG_DATABASE`), defaulting to `localhost:5432` with a `postgres`/`postgres` user for local development.
171
+
172
+ After generation, the CLI prints a short summary — tables and rows generated, relationships found in the DBML, and any columns that fell back to generic placeholder text because neither their type nor name could be matched.
173
+
174
+ Pass `--unit-tests` to also generate deterministic dbt unit test fixtures (`models/staging/ut_stg_<table>.yml`) from the actually-generated seed rows:
175
+
176
+ ```bash
177
+ model2data --file examples/hackernews.dbml --rows 200 --seed 42 --unit-tests
178
+ ```
179
+
180
+ This targets dbt-core's native unit testing feature, which requires **dbt-core >= 1.8**. Since this project only requires `dbt-core>=1.5.0` by default, unit test generation is opt-in — leave the flag off if your dbt-core version is older.
181
+
182
+ ---
183
+
184
+ ## Generated dbt project structure
185
+
186
+ The generated dbt project includes:
187
+
188
+ ```
189
+ dbt_{project_name}/
190
+ ├── seeds/
191
+ │ └── raw/
192
+ │ ├── table1.csv
193
+ │ └── table2.csv
194
+ ├── models/
195
+ │ └── staging/
196
+ │ ├── __sources.yml
197
+ │ ├── stg_table1.sql
198
+ │ ├── stg_table1.yml
199
+ │ ├── ut_stg_table1.yml # only with --unit-tests
200
+ │ └── ...
201
+ ├── data-tests/
202
+ │ └── unique_combination_stg_table1_col_a_col_b.sql # only for composite pk/unique keys
203
+ ├── macros/
204
+ │ └── generate_schema_name.sql
205
+ ├── dbt_project.yml
206
+ ├── profiles.yml # DuckDB or Postgres config, depending on --adapter
207
+ └── {project_name}_profile.duckdb # DuckDB adapter only
208
+ ```
209
+
210
+ - **Seeds**: CSV files with generated synthetic data.
211
+ - **Staging Models**: Basic dbt models that load from seeds.
212
+ - **Sources & Tests**: YAML configs defining sources and column tests (`not_null`, `unique`,
213
+ `relationships`, and `accepted_values` for DBML `Enum`-typed columns). Table and column
214
+ `Note` text from the DBML becomes `description:` fields.
215
+ - **Composite key tests**: Composite primary/unique keys declared in an `indexes { }` block get
216
+ a singular SQL test under `data-tests/`, dbt's configured `test-paths`.
217
+ - **Profiles**: Pre-configured for DuckDB (file-based) or Postgres (via env vars), with schema handling.
218
+ - **Unit tests** (opt-in via `--unit-tests`): `models/staging/ut_stg_<table>.yml` fixtures built
219
+ from real generated rows, co-located with each staging model so dbt (which only parses unit
220
+ tests from `model-paths`) picks them up. Requires dbt-core >= 1.8.
221
+
222
+ ---
223
+
224
+ ## Design decisions / non-goals
225
+
226
+ - **DuckDB Default**: Chosen for its zero-config, file-based nature, making it easy to get started without database setup. Postgres is supported via `--adapter postgres`; other adapters can be configured manually.
227
+ - **dbt Integration**: Leverages dbt's transformation capabilities for a familiar workflow in analytics engineering.
228
+ - **Synthetic Data**: Uses deterministic generation for reproducibility; not intended for production use or as a replacement for real data.
229
+ - **Non-goals**: This is not a data migration tool, ETL pipeline, or real-time data generator. It focuses on static, synthetic datasets for testing and prototyping.
230
+
231
+ ---
232
+
233
+ ## Limitations
234
+
235
+ - Supports basic DBML features; complex constraints or advanced SQL types may not be fully handled.
236
+ - Synthetic data generation is heuristic-based and may not perfectly mimic real-world distributions or edge cases.
237
+ - DuckDB and Postgres are supported today; other databases require manual profile adjustments.
238
+ - No support for incremental models or advanced dbt features in generated projects.
239
+
240
+ ---
241
+
242
+ ## Roadmap
243
+
244
+ - [x] Postgres adapter support (`--adapter postgres`)
245
+ - [x] Name-aware synthetic data (email, name, address, phone, etc. instead of generic text)
246
+ - [x] Post-run generation summary (tables, rows, relationships, unmapped columns)
247
+ - [x] DBML `Enum` support, real table/column notes as dbt descriptions, composite keys from
248
+ `indexes { }`, `default:` values, and opt-in deterministic dbt unit test scaffolding
249
+ (`--unit-tests`)
250
+ - [ ] Additional database adapters (e.g., Snowflake, BigQuery).
251
+ - [ ] Enhanced data type handling and custom generators.
252
+ - [ ] Improved schema exploration and developer tooling.
253
+
254
+ ---
255
+
256
+ ## Contributing
257
+
258
+ We welcome contributions!
259
+
260
+ - Open issues for bugs or feature requests.
261
+ - Submit PRs to add new DBML examples, custom data generators, or improvements.
262
+ - Ensure all new features include tests if possible.
263
+
264
+ See [CONTRIBUTING.md](CONTRIBUTING.md) for detailed guidelines, and [DEVELOPMENT.md](DEVELOPMENT.md) for the local dev setup and release process.
265
+
266
+ ## Code of Conduct
267
+
268
+ Please read our [Code of Conduct](CODE_OF_CONDUCT.md) to understand our community standards.
269
+
270
+ ---
271
+
272
+ ## License
273
+
274
+ MIT License. See LICENSE for details.
275
+
276
+ ---
277
+
278
+ <p align="center">
279
+ <a href="https://www.jbanalytica.com">
280
+ <img src="assets/jba-icon-dark-bg.svg" alt="JB Analytica" height="40">
281
+ </a>
282
+ <br>
283
+ Built and maintained by <a href="https://www.jbanalytica.com"><strong>JB Analytica</strong></a> —
284
+ Data & Analytics Engineering · Data Platform Architecture · Modern BI.
285
+ </p>
@@ -0,0 +1,260 @@
1
+ # model2data
2
+
3
+ [![PyPI](https://img.shields.io/pypi/v/model2data)](https://pypi.org/project/model2data/)
4
+ [![CI](https://github.com/JB-Analytica/model2data/actions/workflows/ci.yml/badge.svg)](https://github.com/JB-Analytica/model2data/actions/workflows/ci.yml)
5
+ [![codecov](https://codecov.io/gh/JB-Analytica/model2data/branch/main/graph/badge.svg)](https://codecov.io/gh/JB-Analytica/model2data)
6
+ [![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
7
+
8
+ **Turn a data model into a running analytics stack in one command.**
9
+
10
+ Give `model2data` a [DBML](https://dbml.dbdiagram.io/docs/) schema — hand-written or exported
11
+ from an existing database — and it generates realistic, relationship-preserving synthetic data
12
+ *and* a complete, runnable dbt project around it: seeds, staging models, sources, tests, and a
13
+ DuckDB or Postgres profile. No sample data to hunt down, no dbt boilerplate to hand-write, no
14
+ production data to risk exposing.
15
+
16
+ ```bash
17
+ pip install model2data
18
+ model2data --file examples/hackernews.dbml --rows 200 --seed 42
19
+ cd dbt_hackernews && dbt seed && dbt run
20
+ ```
21
+
22
+ That's a working analytics stack — real (synthetic) data, tested dbt models, queryable in
23
+ DuckDB — from a schema file, in seconds:
24
+
25
+ ![model2data generating a project and running it with dbt](assets/demo.gif)
26
+
27
+ ---
28
+
29
+ ## Why this exists
30
+
31
+ Analytics engineers hit the same wall constantly: you need realistic data to build or test a
32
+ pipeline, but production data is off-limits (privacy, access, scale), and hand-rolling mock CSVs
33
+ is tedious and doesn't scale past two tables. `model2data` closes that gap — from a schema
34
+ definition to a seeded, tested dbt project you can actually run, with no database or production
35
+ access required.
36
+
37
+ - **Privacy-safe.** Nothing but a schema definition goes in; nothing but synthetic data comes out.
38
+ - **Realistic, not random.** Column names are matched against ~35 common patterns — `email`,
39
+ `first_name`, `city`, `phone`, `company`, ... — so a column called `email` gets real-looking
40
+ emails, not `Lorem ipsum` text.
41
+ - **Relationship-preserving.** Foreign keys resolve to real parent rows; tables are generated in
42
+ dependency order.
43
+ - **Deterministic.** Pass `--seed` and the same schema always produces the same data — safe to
44
+ commit fixtures, safe to diff across CI runs.
45
+ - **A real dbt project, not just CSVs.** Seeds, staging models, sources, schema tests, and a
46
+ ready-to-use profile — the thing you'd otherwise spend an afternoon scaffolding by hand.
47
+
48
+ ## Who is model2data for?
49
+
50
+ - **Analytics engineers** — generate realistic datasets and a working dbt project without
51
+ waiting on production access.
52
+ - **Data engineers** — produce deterministic test data from an existing schema for pipeline and
53
+ migration testing.
54
+ - **Software & data teams** — prototype integrations and analytics workflows without exposing
55
+ production data.
56
+ - **Consultants & architects** — spin up realistic environments for demos, workshops, and
57
+ architecture validation in minutes, not hours.
58
+
59
+ ## How it works
60
+
61
+ ```mermaid
62
+ flowchart LR
63
+ subgraph input [" "]
64
+ A["📄 DBML schema"]
65
+ end
66
+
67
+ subgraph m2d ["model2data"]
68
+ direction LR
69
+ B["Parse\ntables, columns,\nrelationships"] --> C["Generate\nFaker + name-aware\ninference, FK-aware"]
70
+ C --> D["Scaffold\nseeds · staging models\ntests · profile"]
71
+ end
72
+
73
+ subgraph output ["Generated dbt project"]
74
+ direction TB
75
+ E["seeds/*.csv"]
76
+ F["models/staging/*.sql + *.yml"]
77
+ G["profiles.yml\n(DuckDB or Postgres)"]
78
+ end
79
+
80
+ A --> B
81
+ D --> E
82
+ D --> F
83
+ D --> G
84
+ E & F & G --> H["dbt seed && dbt run"]
85
+ H --> I[("Analytics-ready\ndataset")]
86
+
87
+ classDef m2dStyle fill:#0A3866,stroke:#2196F0,color:#F6F8FB
88
+ classDef outStyle fill:#182333,stroke:#A8C9EE,color:#F6F8FB
89
+ classDef endStyle fill:#FA9306,stroke:#FA9306,color:#182333
90
+ class B,C,D m2dStyle
91
+ class E,F,G outStyle
92
+ class H,I endStyle
93
+ ```
94
+
95
+ 1. **Parse.** Reads tables, columns, types, and `Ref` relationships from a DBML file.
96
+ 2. **Generate.** Produces synthetic values per column — typed generation for known SQL types
97
+ (int, date, timestamp, ...), name-aware inference for everything else (`email`, `phone`,
98
+ `city`, ...), foreign keys resolved against already-generated parent rows.
99
+ 3. **Scaffold.** Writes a complete dbt project around that data: CSV seeds, staging models with
100
+ `source`/`not_null`/`unique`/`relationships` tests, `accepted_values` tests for DBML
101
+ `Enum`-typed columns, singular SQL tests for composite primary/unique keys, table and column
102
+ `description:` fields pulled from DBML notes, and a profile for DuckDB (zero-config,
103
+ file-based) or Postgres.
104
+
105
+ ---
106
+
107
+ ## Installation
108
+
109
+ ```bash
110
+ pip install model2data
111
+ ```
112
+
113
+ ---
114
+
115
+ ## Quick start
116
+
117
+ We provide an example Hacker News dataset in `examples/hackernews.dbml`.
118
+
119
+ Generate a project with synthetic data:
120
+
121
+ ```bash
122
+ model2data --file examples/hackernews.dbml --rows 200 --seed 42
123
+ ```
124
+
125
+ This creates a `dbt_hackernews/` folder with your data and dbt setup.
126
+
127
+ Run dbt to load and transform the data:
128
+
129
+ ```bash
130
+ cd dbt_hackernews
131
+ dbt deps
132
+ dbt seed
133
+ dbt run
134
+ ```
135
+
136
+ Your analytics-ready dataset is now in DuckDB!
137
+
138
+ To target Postgres instead, install the extra and pass `--adapter postgres`:
139
+
140
+ ```bash
141
+ pip install "model2data[postgres]"
142
+ model2data --file examples/hackernews.dbml --rows 200 --seed 42 --adapter postgres
143
+ ```
144
+
145
+ Connection details are read from environment variables (`MODEL2DATA_PG_HOST`, `MODEL2DATA_PG_PORT`, `MODEL2DATA_PG_USER`, `MODEL2DATA_PG_PASSWORD`, `MODEL2DATA_PG_DATABASE`), defaulting to `localhost:5432` with a `postgres`/`postgres` user for local development.
146
+
147
+ After generation, the CLI prints a short summary — tables and rows generated, relationships found in the DBML, and any columns that fell back to generic placeholder text because neither their type nor name could be matched.
148
+
149
+ Pass `--unit-tests` to also generate deterministic dbt unit test fixtures (`models/staging/ut_stg_<table>.yml`) from the actually-generated seed rows:
150
+
151
+ ```bash
152
+ model2data --file examples/hackernews.dbml --rows 200 --seed 42 --unit-tests
153
+ ```
154
+
155
+ This targets dbt-core's native unit testing feature, which requires **dbt-core >= 1.8**. Since this project only requires `dbt-core>=1.5.0` by default, unit test generation is opt-in — leave the flag off if your dbt-core version is older.
156
+
157
+ ---
158
+
159
+ ## Generated dbt project structure
160
+
161
+ The generated dbt project includes:
162
+
163
+ ```
164
+ dbt_{project_name}/
165
+ ├── seeds/
166
+ │ └── raw/
167
+ │ ├── table1.csv
168
+ │ └── table2.csv
169
+ ├── models/
170
+ │ └── staging/
171
+ │ ├── __sources.yml
172
+ │ ├── stg_table1.sql
173
+ │ ├── stg_table1.yml
174
+ │ ├── ut_stg_table1.yml # only with --unit-tests
175
+ │ └── ...
176
+ ├── data-tests/
177
+ │ └── unique_combination_stg_table1_col_a_col_b.sql # only for composite pk/unique keys
178
+ ├── macros/
179
+ │ └── generate_schema_name.sql
180
+ ├── dbt_project.yml
181
+ ├── profiles.yml # DuckDB or Postgres config, depending on --adapter
182
+ └── {project_name}_profile.duckdb # DuckDB adapter only
183
+ ```
184
+
185
+ - **Seeds**: CSV files with generated synthetic data.
186
+ - **Staging Models**: Basic dbt models that load from seeds.
187
+ - **Sources & Tests**: YAML configs defining sources and column tests (`not_null`, `unique`,
188
+ `relationships`, and `accepted_values` for DBML `Enum`-typed columns). Table and column
189
+ `Note` text from the DBML becomes `description:` fields.
190
+ - **Composite key tests**: Composite primary/unique keys declared in an `indexes { }` block get
191
+ a singular SQL test under `data-tests/`, dbt's configured `test-paths`.
192
+ - **Profiles**: Pre-configured for DuckDB (file-based) or Postgres (via env vars), with schema handling.
193
+ - **Unit tests** (opt-in via `--unit-tests`): `models/staging/ut_stg_<table>.yml` fixtures built
194
+ from real generated rows, co-located with each staging model so dbt (which only parses unit
195
+ tests from `model-paths`) picks them up. Requires dbt-core >= 1.8.
196
+
197
+ ---
198
+
199
+ ## Design decisions / non-goals
200
+
201
+ - **DuckDB Default**: Chosen for its zero-config, file-based nature, making it easy to get started without database setup. Postgres is supported via `--adapter postgres`; other adapters can be configured manually.
202
+ - **dbt Integration**: Leverages dbt's transformation capabilities for a familiar workflow in analytics engineering.
203
+ - **Synthetic Data**: Uses deterministic generation for reproducibility; not intended for production use or as a replacement for real data.
204
+ - **Non-goals**: This is not a data migration tool, ETL pipeline, or real-time data generator. It focuses on static, synthetic datasets for testing and prototyping.
205
+
206
+ ---
207
+
208
+ ## Limitations
209
+
210
+ - Supports basic DBML features; complex constraints or advanced SQL types may not be fully handled.
211
+ - Synthetic data generation is heuristic-based and may not perfectly mimic real-world distributions or edge cases.
212
+ - DuckDB and Postgres are supported today; other databases require manual profile adjustments.
213
+ - No support for incremental models or advanced dbt features in generated projects.
214
+
215
+ ---
216
+
217
+ ## Roadmap
218
+
219
+ - [x] Postgres adapter support (`--adapter postgres`)
220
+ - [x] Name-aware synthetic data (email, name, address, phone, etc. instead of generic text)
221
+ - [x] Post-run generation summary (tables, rows, relationships, unmapped columns)
222
+ - [x] DBML `Enum` support, real table/column notes as dbt descriptions, composite keys from
223
+ `indexes { }`, `default:` values, and opt-in deterministic dbt unit test scaffolding
224
+ (`--unit-tests`)
225
+ - [ ] Additional database adapters (e.g., Snowflake, BigQuery).
226
+ - [ ] Enhanced data type handling and custom generators.
227
+ - [ ] Improved schema exploration and developer tooling.
228
+
229
+ ---
230
+
231
+ ## Contributing
232
+
233
+ We welcome contributions!
234
+
235
+ - Open issues for bugs or feature requests.
236
+ - Submit PRs to add new DBML examples, custom data generators, or improvements.
237
+ - Ensure all new features include tests if possible.
238
+
239
+ See [CONTRIBUTING.md](CONTRIBUTING.md) for detailed guidelines, and [DEVELOPMENT.md](DEVELOPMENT.md) for the local dev setup and release process.
240
+
241
+ ## Code of Conduct
242
+
243
+ Please read our [Code of Conduct](CODE_OF_CONDUCT.md) to understand our community standards.
244
+
245
+ ---
246
+
247
+ ## License
248
+
249
+ MIT License. See LICENSE for details.
250
+
251
+ ---
252
+
253
+ <p align="center">
254
+ <a href="https://www.jbanalytica.com">
255
+ <img src="assets/jba-icon-dark-bg.svg" alt="JB Analytica" height="40">
256
+ </a>
257
+ <br>
258
+ Built and maintained by <a href="https://www.jbanalytica.com"><strong>JB Analytica</strong></a> —
259
+ Data & Analytics Engineering · Data Platform Architecture · Modern BI.
260
+ </p>
@@ -11,7 +11,7 @@ from model2data.dbt.project import (
11
11
  create_project_scaffold,
12
12
  create_staging_models,
13
13
  )
14
- from model2data.dbt.tests import generate_dbt_yml
14
+ from model2data.dbt.tests import generate_dbt_yml, generate_unit_tests
15
15
  from model2data.generate.core import generate_data_from_dbml
16
16
  from model2data.generate.faker import get_unmapped_columns, reset_stats
17
17
  from model2data.parse.dbml import parse_dbml
@@ -76,6 +76,14 @@ def main(
76
76
  "-a",
77
77
  help=f"dbt warehouse adapter to target. One of: {', '.join(SUPPORTED_ADAPTERS)}.",
78
78
  ),
79
+ unit_tests: bool = typer.Option(
80
+ False,
81
+ "--unit-tests",
82
+ help=(
83
+ "Also generate deterministic dbt unit test fixtures (models/staging/ut_stg_*.yml) "
84
+ "from the generated seed rows. Requires dbt-core >= 1.8 to run."
85
+ ),
86
+ ),
79
87
  ):
80
88
  """
81
89
  Generate synthetic data and a dbt project from a DBML model.
@@ -152,6 +160,10 @@ def main(
152
160
  typer.echo("🧪 Generating dbt yml with tests...")
153
161
  generate_dbt_yml(dest, tables, refs, project_name)
154
162
 
163
+ if unit_tests:
164
+ typer.echo("🔬 Generating dbt unit test fixtures (requires dbt-core >= 1.8)...")
165
+ generate_unit_tests(dest, tables, generated_tables)
166
+
155
167
  typer.echo(f"🪪 Ensuring dbt profile exists ({adapter})...")
156
168
  create_profiles_yml(dest, profile_name, adapter=adapter)
157
169
 
@@ -13,7 +13,7 @@ def create_project_scaffold(dest: Path, project_name: str, profile_name: str) ->
13
13
  (dest / "seeds" / "raw").mkdir(parents=True, exist_ok=True)
14
14
  (dest / "analysis").mkdir(exist_ok=True)
15
15
  (dest / "macros").mkdir(exist_ok=True)
16
- (dest / "tests").mkdir(exist_ok=True)
16
+ (dest / "data-tests").mkdir(exist_ok=True)
17
17
  (dest / "snapshots").mkdir(exist_ok=True)
18
18
 
19
19
  # dbt_project.yml