model2data 0.3.0__tar.gz → 0.4.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- model2data-0.4.3/PKG-INFO +285 -0
- model2data-0.4.3/README.md +260 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data/cli.py +13 -1
- {model2data-0.3.0 → model2data-0.4.3}/model2data/dbt/project.py +1 -1
- model2data-0.4.3/model2data/dbt/tests.py +276 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data/generate/core.py +101 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data/generate/faker.py +69 -3
- model2data-0.4.3/model2data/parse/dbml.py +452 -0
- model2data-0.4.3/model2data.egg-info/PKG-INFO +285 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data.egg-info/SOURCES.txt +1 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data.egg-info/requires.txt +0 -1
- {model2data-0.3.0 → model2data-0.4.3}/pyproject.toml +2 -8
- {model2data-0.3.0 → model2data-0.4.3}/tests/test_cli.py +65 -0
- {model2data-0.3.0 → model2data-0.4.3}/tests/test_dbml_parser.py +540 -7
- model2data-0.4.3/tests/test_dbt_integration.py +104 -0
- {model2data-0.3.0 → model2data-0.4.3}/tests/test_dbt_project.py +1 -1
- model2data-0.4.3/tests/test_dbt_tests.py +348 -0
- model2data-0.4.3/tests/test_faker_name_inference.py +191 -0
- {model2data-0.3.0 → model2data-0.4.3}/tests/test_generation.py +134 -0
- model2data-0.3.0/PKG-INFO +0 -177
- model2data-0.3.0/README.md +0 -151
- model2data-0.3.0/model2data/dbt/tests.py +0 -92
- model2data-0.3.0/model2data/parse/dbml.py +0 -227
- model2data-0.3.0/model2data.egg-info/PKG-INFO +0 -177
- model2data-0.3.0/tests/test_dbt_tests.py +0 -64
- model2data-0.3.0/tests/test_faker_name_inference.py +0 -79
- {model2data-0.3.0 → model2data-0.4.3}/LICENSE +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data/__init__.py +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data/dbt/__init__.py +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data/dbt/templates/dbt_project.yml.jinja +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data/dbt/templates/macros/generate_schema_name.sql +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data/dbt/templates/profiles.yml.jinja +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data/generate/__init__.py +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data/generate/relationships.py +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data/parse/__init__.py +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data/utils.py +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data.egg-info/dependency_links.txt +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data.egg-info/entry_points.txt +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/model2data.egg-info/top_level.txt +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/setup.cfg +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/tests/test_coverage_gaps.py +0 -0
- {model2data-0.3.0 → model2data-0.4.3}/tests/test_dbt_naming.py +0 -0
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: model2data
|
|
3
|
+
Version: 0.4.3
|
|
4
|
+
Summary: Generate analytics-ready datasets from DBML models
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Dist: dbt-core>=1.5.0
|
|
9
|
+
Requires-Dist: dbt-duckdb>=1.5.0
|
|
10
|
+
Requires-Dist: faker>=37.12.0
|
|
11
|
+
Requires-Dist: pandas>=2.3.3
|
|
12
|
+
Requires-Dist: pyyaml>=6.0.3
|
|
13
|
+
Requires-Dist: typer>=0.20.0
|
|
14
|
+
Provides-Extra: postgres
|
|
15
|
+
Requires-Dist: dbt-postgres>=1.5.0; extra == "postgres"
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest; extra == "dev"
|
|
18
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
19
|
+
Requires-Dist: pre-commit; extra == "dev"
|
|
20
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
21
|
+
Requires-Dist: ty>=0.0.4; extra == "dev"
|
|
22
|
+
Requires-Dist: types-pyyaml; extra == "dev"
|
|
23
|
+
Requires-Dist: poethepoet>=0.38.0; extra == "dev"
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# model2data
|
|
27
|
+
|
|
28
|
+
[](https://pypi.org/project/model2data/)
|
|
29
|
+
[](https://github.com/JB-Analytica/model2data/actions/workflows/ci.yml)
|
|
30
|
+
[](https://codecov.io/gh/JB-Analytica/model2data)
|
|
31
|
+
[](LICENSE)
|
|
32
|
+
|
|
33
|
+
**Turn a data model into a running analytics stack in one command.**
|
|
34
|
+
|
|
35
|
+
Give `model2data` a [DBML](https://dbml.dbdiagram.io/docs/) schema — hand-written or exported
|
|
36
|
+
from an existing database — and it generates realistic, relationship-preserving synthetic data
|
|
37
|
+
*and* a complete, runnable dbt project around it: seeds, staging models, sources, tests, and a
|
|
38
|
+
DuckDB or Postgres profile. No sample data to hunt down, no dbt boilerplate to hand-write, no
|
|
39
|
+
production data to risk exposing.
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
pip install model2data
|
|
43
|
+
model2data --file examples/hackernews.dbml --rows 200 --seed 42
|
|
44
|
+
cd dbt_hackernews && dbt seed && dbt run
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
That's a working analytics stack — real (synthetic) data, tested dbt models, queryable in
|
|
48
|
+
DuckDB — from a schema file, in seconds:
|
|
49
|
+
|
|
50
|
+

|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## Why this exists
|
|
55
|
+
|
|
56
|
+
Analytics engineers hit the same wall constantly: you need realistic data to build or test a
|
|
57
|
+
pipeline, but production data is off-limits (privacy, access, scale), and hand-rolling mock CSVs
|
|
58
|
+
is tedious and doesn't scale past two tables. `model2data` closes that gap — from a schema
|
|
59
|
+
definition to a seeded, tested dbt project you can actually run, with no database or production
|
|
60
|
+
access required.
|
|
61
|
+
|
|
62
|
+
- **Privacy-safe.** Nothing but a schema definition goes in; nothing but synthetic data comes out.
|
|
63
|
+
- **Realistic, not random.** Column names are matched against ~35 common patterns — `email`,
|
|
64
|
+
`first_name`, `city`, `phone`, `company`, ... — so a column called `email` gets real-looking
|
|
65
|
+
emails, not `Lorem ipsum` text.
|
|
66
|
+
- **Relationship-preserving.** Foreign keys resolve to real parent rows; tables are generated in
|
|
67
|
+
dependency order.
|
|
68
|
+
- **Deterministic.** Pass `--seed` and the same schema always produces the same data — safe to
|
|
69
|
+
commit fixtures, safe to diff across CI runs.
|
|
70
|
+
- **A real dbt project, not just CSVs.** Seeds, staging models, sources, schema tests, and a
|
|
71
|
+
ready-to-use profile — the thing you'd otherwise spend an afternoon scaffolding by hand.
|
|
72
|
+
|
|
73
|
+
## Who is model2data for?
|
|
74
|
+
|
|
75
|
+
- **Analytics engineers** — generate realistic datasets and a working dbt project without
|
|
76
|
+
waiting on production access.
|
|
77
|
+
- **Data engineers** — produce deterministic test data from an existing schema for pipeline and
|
|
78
|
+
migration testing.
|
|
79
|
+
- **Software & data teams** — prototype integrations and analytics workflows without exposing
|
|
80
|
+
production data.
|
|
81
|
+
- **Consultants & architects** — spin up realistic environments for demos, workshops, and
|
|
82
|
+
architecture validation in minutes, not hours.
|
|
83
|
+
|
|
84
|
+
## How it works
|
|
85
|
+
|
|
86
|
+
```mermaid
|
|
87
|
+
flowchart LR
|
|
88
|
+
subgraph input [" "]
|
|
89
|
+
A["📄 DBML schema"]
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
subgraph m2d ["model2data"]
|
|
93
|
+
direction LR
|
|
94
|
+
B["Parse\ntables, columns,\nrelationships"] --> C["Generate\nFaker + name-aware\ninference, FK-aware"]
|
|
95
|
+
C --> D["Scaffold\nseeds · staging models\ntests · profile"]
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
subgraph output ["Generated dbt project"]
|
|
99
|
+
direction TB
|
|
100
|
+
E["seeds/*.csv"]
|
|
101
|
+
F["models/staging/*.sql + *.yml"]
|
|
102
|
+
G["profiles.yml\n(DuckDB or Postgres)"]
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
A --> B
|
|
106
|
+
D --> E
|
|
107
|
+
D --> F
|
|
108
|
+
D --> G
|
|
109
|
+
E & F & G --> H["dbt seed && dbt run"]
|
|
110
|
+
H --> I[("Analytics-ready\ndataset")]
|
|
111
|
+
|
|
112
|
+
classDef m2dStyle fill:#0A3866,stroke:#2196F0,color:#F6F8FB
|
|
113
|
+
classDef outStyle fill:#182333,stroke:#A8C9EE,color:#F6F8FB
|
|
114
|
+
classDef endStyle fill:#FA9306,stroke:#FA9306,color:#182333
|
|
115
|
+
class B,C,D m2dStyle
|
|
116
|
+
class E,F,G outStyle
|
|
117
|
+
class H,I endStyle
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
1. **Parse.** Reads tables, columns, types, and `Ref` relationships from a DBML file.
|
|
121
|
+
2. **Generate.** Produces synthetic values per column — typed generation for known SQL types
|
|
122
|
+
(int, date, timestamp, ...), name-aware inference for everything else (`email`, `phone`,
|
|
123
|
+
`city`, ...), foreign keys resolved against already-generated parent rows.
|
|
124
|
+
3. **Scaffold.** Writes a complete dbt project around that data: CSV seeds, staging models with
|
|
125
|
+
`source`/`not_null`/`unique`/`relationships` tests, `accepted_values` tests for DBML
|
|
126
|
+
`Enum`-typed columns, singular SQL tests for composite primary/unique keys, table and column
|
|
127
|
+
`description:` fields pulled from DBML notes, and a profile for DuckDB (zero-config,
|
|
128
|
+
file-based) or Postgres.
|
|
129
|
+
|
|
130
|
+
---
|
|
131
|
+
|
|
132
|
+
## Installation
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
pip install model2data
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
---
|
|
139
|
+
|
|
140
|
+
## Quick start
|
|
141
|
+
|
|
142
|
+
We provide an example Hacker News dataset in `examples/hackernews.dbml`.
|
|
143
|
+
|
|
144
|
+
Generate a project with synthetic data:
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
model2data --file examples/hackernews.dbml --rows 200 --seed 42
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
This creates a `dbt_hackernews/` folder with your data and dbt setup.
|
|
151
|
+
|
|
152
|
+
Run dbt to load and transform the data:
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
cd dbt_hackernews
|
|
156
|
+
dbt deps
|
|
157
|
+
dbt seed
|
|
158
|
+
dbt run
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Your analytics-ready dataset is now in DuckDB!
|
|
162
|
+
|
|
163
|
+
To target Postgres instead, install the extra and pass `--adapter postgres`:
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
pip install "model2data[postgres]"
|
|
167
|
+
model2data --file examples/hackernews.dbml --rows 200 --seed 42 --adapter postgres
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Connection details are read from environment variables (`MODEL2DATA_PG_HOST`, `MODEL2DATA_PG_PORT`, `MODEL2DATA_PG_USER`, `MODEL2DATA_PG_PASSWORD`, `MODEL2DATA_PG_DATABASE`), defaulting to `localhost:5432` with a `postgres`/`postgres` user for local development.
|
|
171
|
+
|
|
172
|
+
After generation, the CLI prints a short summary — tables and rows generated, relationships found in the DBML, and any columns that fell back to generic placeholder text because neither their type nor name could be matched.
|
|
173
|
+
|
|
174
|
+
Pass `--unit-tests` to also generate deterministic dbt unit test fixtures (`models/staging/ut_stg_<table>.yml`) from the actually-generated seed rows:
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
model2data --file examples/hackernews.dbml --rows 200 --seed 42 --unit-tests
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
This targets dbt-core's native unit testing feature, which requires **dbt-core >= 1.8**. Since this project only requires `dbt-core>=1.5.0` by default, unit test generation is opt-in — leave the flag off if your dbt-core version is older.
|
|
181
|
+
|
|
182
|
+
---
|
|
183
|
+
|
|
184
|
+
## Generated dbt project structure
|
|
185
|
+
|
|
186
|
+
The generated dbt project includes:
|
|
187
|
+
|
|
188
|
+
```
|
|
189
|
+
dbt_{project_name}/
|
|
190
|
+
├── seeds/
|
|
191
|
+
│ └── raw/
|
|
192
|
+
│ ├── table1.csv
|
|
193
|
+
│ └── table2.csv
|
|
194
|
+
├── models/
|
|
195
|
+
│ └── staging/
|
|
196
|
+
│ ├── __sources.yml
|
|
197
|
+
│ ├── stg_table1.sql
|
|
198
|
+
│ ├── stg_table1.yml
|
|
199
|
+
│ ├── ut_stg_table1.yml # only with --unit-tests
|
|
200
|
+
│ └── ...
|
|
201
|
+
├── data-tests/
|
|
202
|
+
│ └── unique_combination_stg_table1_col_a_col_b.sql # only for composite pk/unique keys
|
|
203
|
+
├── macros/
|
|
204
|
+
│ └── generate_schema_name.sql
|
|
205
|
+
├── dbt_project.yml
|
|
206
|
+
├── profiles.yml # DuckDB or Postgres config, depending on --adapter
|
|
207
|
+
└── {project_name}_profile.duckdb # DuckDB adapter only
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
- **Seeds**: CSV files with generated synthetic data.
|
|
211
|
+
- **Staging Models**: Basic dbt models that load from seeds.
|
|
212
|
+
- **Sources & Tests**: YAML configs defining sources and column tests (`not_null`, `unique`,
|
|
213
|
+
`relationships`, and `accepted_values` for DBML `Enum`-typed columns). Table and column
|
|
214
|
+
`Note` text from the DBML becomes `description:` fields.
|
|
215
|
+
- **Composite key tests**: Composite primary/unique keys declared in an `indexes { }` block get
|
|
216
|
+
a singular SQL test under `data-tests/`, dbt's configured `test-paths`.
|
|
217
|
+
- **Profiles**: Pre-configured for DuckDB (file-based) or Postgres (via env vars), with schema handling.
|
|
218
|
+
- **Unit tests** (opt-in via `--unit-tests`): `models/staging/ut_stg_<table>.yml` fixtures built
|
|
219
|
+
from real generated rows, co-located with each staging model so dbt (which only parses unit
|
|
220
|
+
tests from `model-paths`) picks them up. Requires dbt-core >= 1.8.
|
|
221
|
+
|
|
222
|
+
---
|
|
223
|
+
|
|
224
|
+
## Design decisions / non-goals
|
|
225
|
+
|
|
226
|
+
- **DuckDB Default**: Chosen for its zero-config, file-based nature, making it easy to get started without database setup. Postgres is supported via `--adapter postgres`; other adapters can be configured manually.
|
|
227
|
+
- **dbt Integration**: Leverages dbt's transformation capabilities for a familiar workflow in analytics engineering.
|
|
228
|
+
- **Synthetic Data**: Uses deterministic generation for reproducibility; not intended for production use or as a replacement for real data.
|
|
229
|
+
- **Non-goals**: This is not a data migration tool, ETL pipeline, or real-time data generator. It focuses on static, synthetic datasets for testing and prototyping.
|
|
230
|
+
|
|
231
|
+
---
|
|
232
|
+
|
|
233
|
+
## Limitations
|
|
234
|
+
|
|
235
|
+
- Supports basic DBML features; complex constraints or advanced SQL types may not be fully handled.
|
|
236
|
+
- Synthetic data generation is heuristic-based and may not perfectly mimic real-world distributions or edge cases.
|
|
237
|
+
- DuckDB and Postgres are supported today; other databases require manual profile adjustments.
|
|
238
|
+
- No support for incremental models or advanced dbt features in generated projects.
|
|
239
|
+
|
|
240
|
+
---
|
|
241
|
+
|
|
242
|
+
## Roadmap
|
|
243
|
+
|
|
244
|
+
- [x] Postgres adapter support (`--adapter postgres`)
|
|
245
|
+
- [x] Name-aware synthetic data (email, name, address, phone, etc. instead of generic text)
|
|
246
|
+
- [x] Post-run generation summary (tables, rows, relationships, unmapped columns)
|
|
247
|
+
- [x] DBML `Enum` support, real table/column notes as dbt descriptions, composite keys from
|
|
248
|
+
`indexes { }`, `default:` values, and opt-in deterministic dbt unit test scaffolding
|
|
249
|
+
(`--unit-tests`)
|
|
250
|
+
- [ ] Additional database adapters (e.g., Snowflake, BigQuery).
|
|
251
|
+
- [ ] Enhanced data type handling and custom generators.
|
|
252
|
+
- [ ] Improved schema exploration and developer tooling.
|
|
253
|
+
|
|
254
|
+
---
|
|
255
|
+
|
|
256
|
+
## Contributing
|
|
257
|
+
|
|
258
|
+
We welcome contributions!
|
|
259
|
+
|
|
260
|
+
- Open issues for bugs or feature requests.
|
|
261
|
+
- Submit PRs to add new DBML examples, custom data generators, or improvements.
|
|
262
|
+
- Ensure all new features include tests if possible.
|
|
263
|
+
|
|
264
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for detailed guidelines, and [DEVELOPMENT.md](DEVELOPMENT.md) for the local dev setup and release process.
|
|
265
|
+
|
|
266
|
+
## Code of Conduct
|
|
267
|
+
|
|
268
|
+
Please read our [Code of Conduct](CODE_OF_CONDUCT.md) to understand our community standards.
|
|
269
|
+
|
|
270
|
+
---
|
|
271
|
+
|
|
272
|
+
## License
|
|
273
|
+
|
|
274
|
+
MIT License. See LICENSE for details.
|
|
275
|
+
|
|
276
|
+
---
|
|
277
|
+
|
|
278
|
+
<p align="center">
|
|
279
|
+
<a href="https://www.jbanalytica.com">
|
|
280
|
+
<img src="assets/jba-icon-dark-bg.svg" alt="JB Analytica" height="40">
|
|
281
|
+
</a>
|
|
282
|
+
<br>
|
|
283
|
+
Built and maintained by <a href="https://www.jbanalytica.com"><strong>JB Analytica</strong></a> —
|
|
284
|
+
Data & Analytics Engineering · Data Platform Architecture · Modern BI.
|
|
285
|
+
</p>
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
# model2data
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/model2data/)
|
|
4
|
+
[](https://github.com/JB-Analytica/model2data/actions/workflows/ci.yml)
|
|
5
|
+
[](https://codecov.io/gh/JB-Analytica/model2data)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+
**Turn a data model into a running analytics stack in one command.**
|
|
9
|
+
|
|
10
|
+
Give `model2data` a [DBML](https://dbml.dbdiagram.io/docs/) schema — hand-written or exported
|
|
11
|
+
from an existing database — and it generates realistic, relationship-preserving synthetic data
|
|
12
|
+
*and* a complete, runnable dbt project around it: seeds, staging models, sources, tests, and a
|
|
13
|
+
DuckDB or Postgres profile. No sample data to hunt down, no dbt boilerplate to hand-write, no
|
|
14
|
+
production data to risk exposing.
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install model2data
|
|
18
|
+
model2data --file examples/hackernews.dbml --rows 200 --seed 42
|
|
19
|
+
cd dbt_hackernews && dbt seed && dbt run
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
That's a working analytics stack — real (synthetic) data, tested dbt models, queryable in
|
|
23
|
+
DuckDB — from a schema file, in seconds:
|
|
24
|
+
|
|
25
|
+

|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## Why this exists
|
|
30
|
+
|
|
31
|
+
Analytics engineers hit the same wall constantly: you need realistic data to build or test a
|
|
32
|
+
pipeline, but production data is off-limits (privacy, access, scale), and hand-rolling mock CSVs
|
|
33
|
+
is tedious and doesn't scale past two tables. `model2data` closes that gap — from a schema
|
|
34
|
+
definition to a seeded, tested dbt project you can actually run, with no database or production
|
|
35
|
+
access required.
|
|
36
|
+
|
|
37
|
+
- **Privacy-safe.** Nothing but a schema definition goes in; nothing but synthetic data comes out.
|
|
38
|
+
- **Realistic, not random.** Column names are matched against ~35 common patterns — `email`,
|
|
39
|
+
`first_name`, `city`, `phone`, `company`, ... — so a column called `email` gets real-looking
|
|
40
|
+
emails, not `Lorem ipsum` text.
|
|
41
|
+
- **Relationship-preserving.** Foreign keys resolve to real parent rows; tables are generated in
|
|
42
|
+
dependency order.
|
|
43
|
+
- **Deterministic.** Pass `--seed` and the same schema always produces the same data — safe to
|
|
44
|
+
commit fixtures, safe to diff across CI runs.
|
|
45
|
+
- **A real dbt project, not just CSVs.** Seeds, staging models, sources, schema tests, and a
|
|
46
|
+
ready-to-use profile — the thing you'd otherwise spend an afternoon scaffolding by hand.
|
|
47
|
+
|
|
48
|
+
## Who is model2data for?
|
|
49
|
+
|
|
50
|
+
- **Analytics engineers** — generate realistic datasets and a working dbt project without
|
|
51
|
+
waiting on production access.
|
|
52
|
+
- **Data engineers** — produce deterministic test data from an existing schema for pipeline and
|
|
53
|
+
migration testing.
|
|
54
|
+
- **Software & data teams** — prototype integrations and analytics workflows without exposing
|
|
55
|
+
production data.
|
|
56
|
+
- **Consultants & architects** — spin up realistic environments for demos, workshops, and
|
|
57
|
+
architecture validation in minutes, not hours.
|
|
58
|
+
|
|
59
|
+
## How it works
|
|
60
|
+
|
|
61
|
+
```mermaid
|
|
62
|
+
flowchart LR
|
|
63
|
+
subgraph input [" "]
|
|
64
|
+
A["📄 DBML schema"]
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
subgraph m2d ["model2data"]
|
|
68
|
+
direction LR
|
|
69
|
+
B["Parse\ntables, columns,\nrelationships"] --> C["Generate\nFaker + name-aware\ninference, FK-aware"]
|
|
70
|
+
C --> D["Scaffold\nseeds · staging models\ntests · profile"]
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
subgraph output ["Generated dbt project"]
|
|
74
|
+
direction TB
|
|
75
|
+
E["seeds/*.csv"]
|
|
76
|
+
F["models/staging/*.sql + *.yml"]
|
|
77
|
+
G["profiles.yml\n(DuckDB or Postgres)"]
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
A --> B
|
|
81
|
+
D --> E
|
|
82
|
+
D --> F
|
|
83
|
+
D --> G
|
|
84
|
+
E & F & G --> H["dbt seed && dbt run"]
|
|
85
|
+
H --> I[("Analytics-ready\ndataset")]
|
|
86
|
+
|
|
87
|
+
classDef m2dStyle fill:#0A3866,stroke:#2196F0,color:#F6F8FB
|
|
88
|
+
classDef outStyle fill:#182333,stroke:#A8C9EE,color:#F6F8FB
|
|
89
|
+
classDef endStyle fill:#FA9306,stroke:#FA9306,color:#182333
|
|
90
|
+
class B,C,D m2dStyle
|
|
91
|
+
class E,F,G outStyle
|
|
92
|
+
class H,I endStyle
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
1. **Parse.** Reads tables, columns, types, and `Ref` relationships from a DBML file.
|
|
96
|
+
2. **Generate.** Produces synthetic values per column — typed generation for known SQL types
|
|
97
|
+
(int, date, timestamp, ...), name-aware inference for everything else (`email`, `phone`,
|
|
98
|
+
`city`, ...), foreign keys resolved against already-generated parent rows.
|
|
99
|
+
3. **Scaffold.** Writes a complete dbt project around that data: CSV seeds, staging models with
|
|
100
|
+
`source`/`not_null`/`unique`/`relationships` tests, `accepted_values` tests for DBML
|
|
101
|
+
`Enum`-typed columns, singular SQL tests for composite primary/unique keys, table and column
|
|
102
|
+
`description:` fields pulled from DBML notes, and a profile for DuckDB (zero-config,
|
|
103
|
+
file-based) or Postgres.
|
|
104
|
+
|
|
105
|
+
---
|
|
106
|
+
|
|
107
|
+
## Installation
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
pip install model2data
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
---
|
|
114
|
+
|
|
115
|
+
## Quick start
|
|
116
|
+
|
|
117
|
+
We provide an example Hacker News dataset in `examples/hackernews.dbml`.
|
|
118
|
+
|
|
119
|
+
Generate a project with synthetic data:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
model2data --file examples/hackernews.dbml --rows 200 --seed 42
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
This creates a `dbt_hackernews/` folder with your data and dbt setup.
|
|
126
|
+
|
|
127
|
+
Run dbt to load and transform the data:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
cd dbt_hackernews
|
|
131
|
+
dbt deps
|
|
132
|
+
dbt seed
|
|
133
|
+
dbt run
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
Your analytics-ready dataset is now in DuckDB!
|
|
137
|
+
|
|
138
|
+
To target Postgres instead, install the extra and pass `--adapter postgres`:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
pip install "model2data[postgres]"
|
|
142
|
+
model2data --file examples/hackernews.dbml --rows 200 --seed 42 --adapter postgres
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Connection details are read from environment variables (`MODEL2DATA_PG_HOST`, `MODEL2DATA_PG_PORT`, `MODEL2DATA_PG_USER`, `MODEL2DATA_PG_PASSWORD`, `MODEL2DATA_PG_DATABASE`), defaulting to `localhost:5432` with a `postgres`/`postgres` user for local development.
|
|
146
|
+
|
|
147
|
+
After generation, the CLI prints a short summary — tables and rows generated, relationships found in the DBML, and any columns that fell back to generic placeholder text because neither their type nor name could be matched.
|
|
148
|
+
|
|
149
|
+
Pass `--unit-tests` to also generate deterministic dbt unit test fixtures (`models/staging/ut_stg_<table>.yml`) from the actually-generated seed rows:
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
model2data --file examples/hackernews.dbml --rows 200 --seed 42 --unit-tests
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
This targets dbt-core's native unit testing feature, which requires **dbt-core >= 1.8**. Since this project only requires `dbt-core>=1.5.0` by default, unit test generation is opt-in — leave the flag off if your dbt-core version is older.
|
|
156
|
+
|
|
157
|
+
---
|
|
158
|
+
|
|
159
|
+
## Generated dbt project structure
|
|
160
|
+
|
|
161
|
+
The generated dbt project includes:
|
|
162
|
+
|
|
163
|
+
```
|
|
164
|
+
dbt_{project_name}/
|
|
165
|
+
├── seeds/
|
|
166
|
+
│ └── raw/
|
|
167
|
+
│ ├── table1.csv
|
|
168
|
+
│ └── table2.csv
|
|
169
|
+
├── models/
|
|
170
|
+
│ └── staging/
|
|
171
|
+
│ ├── __sources.yml
|
|
172
|
+
│ ├── stg_table1.sql
|
|
173
|
+
│ ├── stg_table1.yml
|
|
174
|
+
│ ├── ut_stg_table1.yml # only with --unit-tests
|
|
175
|
+
│ └── ...
|
|
176
|
+
├── data-tests/
|
|
177
|
+
│ └── unique_combination_stg_table1_col_a_col_b.sql # only for composite pk/unique keys
|
|
178
|
+
├── macros/
|
|
179
|
+
│ └── generate_schema_name.sql
|
|
180
|
+
├── dbt_project.yml
|
|
181
|
+
├── profiles.yml # DuckDB or Postgres config, depending on --adapter
|
|
182
|
+
└── {project_name}_profile.duckdb # DuckDB adapter only
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
- **Seeds**: CSV files with generated synthetic data.
|
|
186
|
+
- **Staging Models**: Basic dbt models that load from seeds.
|
|
187
|
+
- **Sources & Tests**: YAML configs defining sources and column tests (`not_null`, `unique`,
|
|
188
|
+
`relationships`, and `accepted_values` for DBML `Enum`-typed columns). Table and column
|
|
189
|
+
`Note` text from the DBML becomes `description:` fields.
|
|
190
|
+
- **Composite key tests**: Composite primary/unique keys declared in an `indexes { }` block get
|
|
191
|
+
a singular SQL test under `data-tests/`, dbt's configured `test-paths`.
|
|
192
|
+
- **Profiles**: Pre-configured for DuckDB (file-based) or Postgres (via env vars), with schema handling.
|
|
193
|
+
- **Unit tests** (opt-in via `--unit-tests`): `models/staging/ut_stg_<table>.yml` fixtures built
|
|
194
|
+
from real generated rows, co-located with each staging model so dbt (which only parses unit
|
|
195
|
+
tests from `model-paths`) picks them up. Requires dbt-core >= 1.8.
|
|
196
|
+
|
|
197
|
+
---
|
|
198
|
+
|
|
199
|
+
## Design decisions / non-goals
|
|
200
|
+
|
|
201
|
+
- **DuckDB Default**: Chosen for its zero-config, file-based nature, making it easy to get started without database setup. Postgres is supported via `--adapter postgres`; other adapters can be configured manually.
|
|
202
|
+
- **dbt Integration**: Leverages dbt's transformation capabilities for a familiar workflow in analytics engineering.
|
|
203
|
+
- **Synthetic Data**: Uses deterministic generation for reproducibility; not intended for production use or as a replacement for real data.
|
|
204
|
+
- **Non-goals**: This is not a data migration tool, ETL pipeline, or real-time data generator. It focuses on static, synthetic datasets for testing and prototyping.
|
|
205
|
+
|
|
206
|
+
---
|
|
207
|
+
|
|
208
|
+
## Limitations
|
|
209
|
+
|
|
210
|
+
- Supports basic DBML features; complex constraints or advanced SQL types may not be fully handled.
|
|
211
|
+
- Synthetic data generation is heuristic-based and may not perfectly mimic real-world distributions or edge cases.
|
|
212
|
+
- DuckDB and Postgres are supported today; other databases require manual profile adjustments.
|
|
213
|
+
- No support for incremental models or advanced dbt features in generated projects.
|
|
214
|
+
|
|
215
|
+
---
|
|
216
|
+
|
|
217
|
+
## Roadmap
|
|
218
|
+
|
|
219
|
+
- [x] Postgres adapter support (`--adapter postgres`)
|
|
220
|
+
- [x] Name-aware synthetic data (email, name, address, phone, etc. instead of generic text)
|
|
221
|
+
- [x] Post-run generation summary (tables, rows, relationships, unmapped columns)
|
|
222
|
+
- [x] DBML `Enum` support, real table/column notes as dbt descriptions, composite keys from
|
|
223
|
+
`indexes { }`, `default:` values, and opt-in deterministic dbt unit test scaffolding
|
|
224
|
+
(`--unit-tests`)
|
|
225
|
+
- [ ] Additional database adapters (e.g., Snowflake, BigQuery).
|
|
226
|
+
- [ ] Enhanced data type handling and custom generators.
|
|
227
|
+
- [ ] Improved schema exploration and developer tooling.
|
|
228
|
+
|
|
229
|
+
---
|
|
230
|
+
|
|
231
|
+
## Contributing
|
|
232
|
+
|
|
233
|
+
We welcome contributions!
|
|
234
|
+
|
|
235
|
+
- Open issues for bugs or feature requests.
|
|
236
|
+
- Submit PRs to add new DBML examples, custom data generators, or improvements.
|
|
237
|
+
- Ensure all new features include tests if possible.
|
|
238
|
+
|
|
239
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for detailed guidelines, and [DEVELOPMENT.md](DEVELOPMENT.md) for the local dev setup and release process.
|
|
240
|
+
|
|
241
|
+
## Code of Conduct
|
|
242
|
+
|
|
243
|
+
Please read our [Code of Conduct](CODE_OF_CONDUCT.md) to understand our community standards.
|
|
244
|
+
|
|
245
|
+
---
|
|
246
|
+
|
|
247
|
+
## License
|
|
248
|
+
|
|
249
|
+
MIT License. See LICENSE for details.
|
|
250
|
+
|
|
251
|
+
---
|
|
252
|
+
|
|
253
|
+
<p align="center">
|
|
254
|
+
<a href="https://www.jbanalytica.com">
|
|
255
|
+
<img src="assets/jba-icon-dark-bg.svg" alt="JB Analytica" height="40">
|
|
256
|
+
</a>
|
|
257
|
+
<br>
|
|
258
|
+
Built and maintained by <a href="https://www.jbanalytica.com"><strong>JB Analytica</strong></a> —
|
|
259
|
+
Data & Analytics Engineering · Data Platform Architecture · Modern BI.
|
|
260
|
+
</p>
|
|
@@ -11,7 +11,7 @@ from model2data.dbt.project import (
|
|
|
11
11
|
create_project_scaffold,
|
|
12
12
|
create_staging_models,
|
|
13
13
|
)
|
|
14
|
-
from model2data.dbt.tests import generate_dbt_yml
|
|
14
|
+
from model2data.dbt.tests import generate_dbt_yml, generate_unit_tests
|
|
15
15
|
from model2data.generate.core import generate_data_from_dbml
|
|
16
16
|
from model2data.generate.faker import get_unmapped_columns, reset_stats
|
|
17
17
|
from model2data.parse.dbml import parse_dbml
|
|
@@ -76,6 +76,14 @@ def main(
|
|
|
76
76
|
"-a",
|
|
77
77
|
help=f"dbt warehouse adapter to target. One of: {', '.join(SUPPORTED_ADAPTERS)}.",
|
|
78
78
|
),
|
|
79
|
+
unit_tests: bool = typer.Option(
|
|
80
|
+
False,
|
|
81
|
+
"--unit-tests",
|
|
82
|
+
help=(
|
|
83
|
+
"Also generate deterministic dbt unit test fixtures (models/staging/ut_stg_*.yml) "
|
|
84
|
+
"from the generated seed rows. Requires dbt-core >= 1.8 to run."
|
|
85
|
+
),
|
|
86
|
+
),
|
|
79
87
|
):
|
|
80
88
|
"""
|
|
81
89
|
Generate synthetic data and a dbt project from a DBML model.
|
|
@@ -152,6 +160,10 @@ def main(
|
|
|
152
160
|
typer.echo("🧪 Generating dbt yml with tests...")
|
|
153
161
|
generate_dbt_yml(dest, tables, refs, project_name)
|
|
154
162
|
|
|
163
|
+
if unit_tests:
|
|
164
|
+
typer.echo("🔬 Generating dbt unit test fixtures (requires dbt-core >= 1.8)...")
|
|
165
|
+
generate_unit_tests(dest, tables, generated_tables)
|
|
166
|
+
|
|
155
167
|
typer.echo(f"🪪 Ensuring dbt profile exists ({adapter})...")
|
|
156
168
|
create_profiles_yml(dest, profile_name, adapter=adapter)
|
|
157
169
|
|
|
@@ -13,7 +13,7 @@ def create_project_scaffold(dest: Path, project_name: str, profile_name: str) ->
|
|
|
13
13
|
(dest / "seeds" / "raw").mkdir(parents=True, exist_ok=True)
|
|
14
14
|
(dest / "analysis").mkdir(exist_ok=True)
|
|
15
15
|
(dest / "macros").mkdir(exist_ok=True)
|
|
16
|
-
(dest / "tests").mkdir(exist_ok=True)
|
|
16
|
+
(dest / "data-tests").mkdir(exist_ok=True)
|
|
17
17
|
(dest / "snapshots").mkdir(exist_ok=True)
|
|
18
18
|
|
|
19
19
|
# dbt_project.yml
|