model2data 1.8.0__tar.gz → 1.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {model2data-1.8.0/model2data.egg-info → model2data-1.9.0}/PKG-INFO +51 -2
- {model2data-1.8.0 → model2data-1.9.0}/README.md +50 -1
- {model2data-1.8.0 → model2data-1.9.0}/README_PYPI.md +50 -1
- {model2data-1.8.0 → model2data-1.9.0}/model2data/cli.py +101 -51
- model2data-1.9.0/model2data/dbt/naming.py +61 -0
- model2data-1.9.0/model2data/dbt/templates/history_macros/model2data_history_tests.sql +38 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/tests.py +253 -55
- model2data-1.9.0/model2data/defects/__init__.py +59 -0
- model2data-1.9.0/model2data/defects/apply.py +1064 -0
- model2data-1.9.0/model2data/defects/checks.py +179 -0
- model2data-1.9.0/model2data/defects/presets.py +311 -0
- model2data-1.9.0/model2data/defects/report.py +245 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/days.py +9 -4
- model2data-1.9.0/model2data/generate/history.py +152 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/model/__init__.py +7 -1
- {model2data-1.8.0 → model2data-1.9.0}/model2data/model/document.py +50 -1
- {model2data-1.8.0 → model2data-1.9.0}/model2data/model/dump.py +15 -3
- {model2data-1.8.0 → model2data-1.9.0}/model2data/model/types.py +39 -1
- {model2data-1.8.0 → model2data-1.9.0}/model2data/model/validate.py +302 -7
- model2data-1.9.0/model2data/output.py +113 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/spec/PUBLISHING.md +13 -9
- {model2data-1.8.0 → model2data-1.9.0}/model2data/spec/README.md +92 -3
- {model2data-1.8.0 → model2data-1.9.0}/model2data/spec/model.schema.json +59 -5
- {model2data-1.8.0 → model2data-1.9.0/model2data.egg-info}/PKG-INFO +51 -2
- {model2data-1.8.0 → model2data-1.9.0}/model2data.egg-info/SOURCES.txt +15 -0
- {model2data-1.8.0 → model2data-1.9.0}/pyproject.toml +1 -1
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_dbt_tests.py +44 -0
- model2data-1.9.0/tests/test_defects.py +772 -0
- model2data-1.9.0/tests/test_defects_dbt.py +238 -0
- model2data-1.9.0/tests/test_defects_replay.py +138 -0
- model2data-1.9.0/tests/test_defects_report.py +370 -0
- model2data-1.9.0/tests/test_defects_review.py +302 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_hint_tests.py +12 -1
- model2data-1.9.0/tests/test_history.py +254 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_model_edges.py +2 -1
- model2data-1.9.0/tests/test_model_messages.py +603 -0
- model2data-1.8.0/tests/test_model_messages.py +0 -250
- {model2data-1.8.0 → model2data-1.9.0}/LICENSE +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/LICENSE +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/VENDORED.md +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/base.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/column.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/enum.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/expression.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/index.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/note.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/project.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/reference.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/sticky_note.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/table.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/table_group.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/classes/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/constants.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/database.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/column.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/common.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/enum.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/generic.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/index.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/project.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/records.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/reference.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/sticky_note.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/table.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/table_group.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/table_partial.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/exceptions.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/parser/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/parser/blueprints.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/parser/parser.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/base.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/column.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/enum.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/expression.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/index.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/note.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/project.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/reference.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/renderer.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/sticky_note.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/table.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/table_group.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/utils.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/column.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/enum.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/expression.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/index.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/note.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/reference.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/renderer.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/table.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/utils.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/tools.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/hint_tests.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/project.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/templates/dbt_project.yml.jinja +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/templates/hint_macros/model2data_hint_tests.sql +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/templates/macros/generate_schema_name.sql +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/templates/profiles.yml.jinja +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/core.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/faker.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/hints.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/kinds.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/options.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/relationships.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/timeline.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/model/_yaml.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/model/dbml.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/model/engine.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/model/errors.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/model/reader.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/parse/__init__.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/parse/dbml.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/spec/examples/coffee_webshop.model2data.yml +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data/utils.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data.egg-info/dependency_links.txt +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data.egg-info/entry_points.txt +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data.egg-info/requires.txt +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/model2data.egg-info/top_level.txt +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/setup.cfg +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_as_of_anchor.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_cli.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_cli_model.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_column_time_hints.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_coverage_gaps.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_days.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_dbml_parser.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_dbml_parser_fuzz.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_dbt_integration.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_dbt_naming.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_dbt_project.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_distributions.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_examples_model.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_faker_name_inference.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_generation.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_lone_country.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_model.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_model_dbml.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_model_vocabulary.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_options.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_release_stress.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_reproducibility.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_row_identity.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_shaping.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_spec_alignment.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_spec_schema.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_table_seeds.py +0 -0
- {model2data-1.8.0 → model2data-1.9.0}/tests/test_timeline.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: model2data
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.9.0
|
|
4
4
|
Summary: Generate analytics-ready datasets from DBML models
|
|
5
5
|
Author: JB Analytica
|
|
6
6
|
License-Expression: MIT
|
|
@@ -132,7 +132,7 @@ access required.
|
|
|
132
132
|
## The model file
|
|
133
133
|
|
|
134
134
|
A model is one YAML document, `<name>.model2data.yml` — or the same document as JSON. Its format
|
|
135
|
-
is [spec 0.
|
|
135
|
+
is [spec 0.3.0](https://github.com/JB-Analytica/model2data/blob/main/model2data/spec/README.md), with a JSON Schema your editor can autocomplete and
|
|
136
136
|
check against:
|
|
137
137
|
|
|
138
138
|
```yaml
|
|
@@ -330,6 +330,55 @@ days[3].tables["orders"].updated # rows day 3 changed, with their new values
|
|
|
330
330
|
days[3].tables["orders"].state # the table after day 3
|
|
331
331
|
```
|
|
332
332
|
|
|
333
|
+
### Break the data on purpose
|
|
334
|
+
|
|
335
|
+
A dbt test you have never seen fail is a test you hope works. `--defects` puts deliberate,
|
|
336
|
+
counted defects in the generated data, so `dbt build` fails exactly the tests it should — for
|
|
337
|
+
teaching dbt, or for proving a project's tests fire:
|
|
338
|
+
|
|
339
|
+
```bash
|
|
340
|
+
model2data --file examples/ecommerce.model2data.yml --seed 42 --defects training
|
|
341
|
+
cd dbt_ecommerce && dbt build # fails unique, not_null, relationships, accepted_values once each
|
|
342
|
+
```
|
|
343
|
+
|
|
344
|
+
`training` breaks each kind of standard dbt test once; `messy` puts a small share of every defect
|
|
345
|
+
on every table; `clean` (the default) is the data as it always was, byte for byte; `none` also
|
|
346
|
+
ignores the defects a model's tables list, for the clean data of a training model. A table can
|
|
347
|
+
also list its own defects in the model (spec 0.3.0):
|
|
348
|
+
|
|
349
|
+
```yaml
|
|
350
|
+
orders:
|
|
351
|
+
defects:
|
|
352
|
+
- {type: duplicate_keys, count: 3} # fails unique
|
|
353
|
+
- {type: orphan_foreign_keys, column: customer_id, share: 0.02} # fails relationships
|
|
354
|
+
- {type: nulls, column: order_date, count: 5} # fails not_null
|
|
355
|
+
```
|
|
356
|
+
|
|
357
|
+
The types are `duplicate_keys`, `orphan_foreign_keys`, `nulls`, `invalid_values` (fails
|
|
358
|
+
`accepted_values`), `messy_text` (whitespace and casing to clean in staging), and, with `--days`,
|
|
359
|
+
`late_arriving` (rows an incremental model filtering on `updated_at > max(updated_at)` skips) and
|
|
360
|
+
`late_updates` (versions a `strategy: timestamp` snapshot misses) and `overlapping_history` (see
|
|
361
|
+
below). Every run with defects writes
|
|
362
|
+
`defects_report.json` (each defect, the rows it broke, the tests it breaks) and
|
|
363
|
+
`EXPECTED_FAILURES.md` into the project, so a test that does not fire is visible. Defects are applied
|
|
364
|
+
after the clean data, from their own random stream: the same seed gives the same bytes, and a
|
|
365
|
+
defect on one table moves nothing in another. From Python, `model2data.defects.planned_defects`
|
|
366
|
+
says what a run will break, `model2data.output.finish_run` applies it (and builds the histories)
|
|
367
|
+
after `generate_days`, and `write_defects_report` / `write_expected_failures` write the two files.
|
|
368
|
+
|
|
369
|
+
### A source that keeps its history
|
|
370
|
+
|
|
371
|
+
`history: true` in a table's `incremental` (spec 0.3.0) also writes `<table>_history`: every
|
|
372
|
+
version of every row over the generated days, with `valid_from`, `valid_to` (null for the current
|
|
373
|
+
version) and `is_current` — an SCD type 2 source to build snapshots and point-in-time joins
|
|
374
|
+
against. The project tests it with two package-free tests, one current row per key and no
|
|
375
|
+
overlapping versions; the `overlapping_history` defect breaks the second.
|
|
376
|
+
|
|
377
|
+
```yaml
|
|
378
|
+
orders:
|
|
379
|
+
incremental: {new_per_day: 20, update_rate: 0.1, updated_at: updated_at, history: true}
|
|
380
|
+
```
|
|
381
|
+
|
|
333
382
|
## Generated dbt project structure
|
|
334
383
|
|
|
335
384
|
The generated dbt project includes:
|
|
@@ -123,7 +123,7 @@ flowchart LR
|
|
|
123
123
|
## The model file
|
|
124
124
|
|
|
125
125
|
A model is one YAML document, `<name>.model2data.yml` — or the same document as JSON. Its
|
|
126
|
-
format is [spec 0.
|
|
126
|
+
format is [spec 0.3.0](model2data/spec/README.md), with a JSON Schema
|
|
127
127
|
([`model.schema.json`](model2data/spec/model.schema.json)) your editor can autocomplete and check
|
|
128
128
|
against:
|
|
129
129
|
|
|
@@ -473,6 +473,55 @@ they describe intent and should not break a first `dbt build` on real data); the
|
|
|
473
473
|
same YAML as before. To call the mapping from Python, `model2data.dbt.hint_tests.hint_tests_for(model)`
|
|
474
474
|
returns the tests as data.
|
|
475
475
|
|
|
476
|
+
### Break the data on purpose
|
|
477
|
+
|
|
478
|
+
A dbt test you have never seen fail is a test you hope works. `--defects` puts deliberate,
|
|
479
|
+
counted defects in the generated data, so `dbt build` fails exactly the tests it should — for
|
|
480
|
+
teaching dbt, or for proving a project's tests fire:
|
|
481
|
+
|
|
482
|
+
```bash
|
|
483
|
+
model2data --file examples/ecommerce.model2data.yml --seed 42 --defects training
|
|
484
|
+
cd dbt_ecommerce && dbt build # fails unique, not_null, relationships, accepted_values once each
|
|
485
|
+
```
|
|
486
|
+
|
|
487
|
+
`training` breaks each kind of standard dbt test once; `messy` puts a small share of every defect
|
|
488
|
+
on every table; `clean` (the default) is the data as it always was, byte for byte; `none` also
|
|
489
|
+
ignores the defects a model's tables list, for the clean data of a training model. A table can
|
|
490
|
+
also list its own defects in the model (spec 0.3.0):
|
|
491
|
+
|
|
492
|
+
```yaml
|
|
493
|
+
orders:
|
|
494
|
+
defects:
|
|
495
|
+
- {type: duplicate_keys, count: 3} # fails unique
|
|
496
|
+
- {type: orphan_foreign_keys, column: customer_id, share: 0.02} # fails relationships
|
|
497
|
+
- {type: nulls, column: order_date, count: 5} # fails not_null
|
|
498
|
+
```
|
|
499
|
+
|
|
500
|
+
The types are `duplicate_keys`, `orphan_foreign_keys`, `nulls`, `invalid_values` (fails
|
|
501
|
+
`accepted_values`), `messy_text` (whitespace and casing to clean in staging), and, with `--days`,
|
|
502
|
+
`late_arriving` (rows an incremental model filtering on `updated_at > max(updated_at)` skips) and
|
|
503
|
+
`late_updates` (versions a `strategy: timestamp` snapshot misses) and `overlapping_history` (see
|
|
504
|
+
below). Every run with defects writes
|
|
505
|
+
`defects_report.json` (each defect, the rows it broke, the tests it breaks) and
|
|
506
|
+
`EXPECTED_FAILURES.md` into the project, so a test that does not fire is visible. Defects are applied
|
|
507
|
+
after the clean data, from their own random stream: the same seed gives the same bytes, and a
|
|
508
|
+
defect on one table moves nothing in another. From Python, `model2data.defects.planned_defects`
|
|
509
|
+
says what a run will break, `model2data.output.finish_run` applies it (and builds the histories)
|
|
510
|
+
after `generate_days`, and `write_defects_report` / `write_expected_failures` write the two files.
|
|
511
|
+
|
|
512
|
+
### A source that keeps its history
|
|
513
|
+
|
|
514
|
+
`history: true` in a table's `incremental` (spec 0.3.0) also writes `<table>_history`: every
|
|
515
|
+
version of every row over the generated days, with `valid_from`, `valid_to` (null for the current
|
|
516
|
+
version) and `is_current` — an SCD type 2 source to build snapshots and point-in-time joins
|
|
517
|
+
against. The project tests it with two package-free tests, one current row per key and no
|
|
518
|
+
overlapping versions; the `overlapping_history` defect breaks the second.
|
|
519
|
+
|
|
520
|
+
```yaml
|
|
521
|
+
orders:
|
|
522
|
+
incremental: {new_per_day: 20, update_rate: 0.1, updated_at: updated_at, history: true}
|
|
523
|
+
```
|
|
524
|
+
|
|
476
525
|
---
|
|
477
526
|
|
|
478
527
|
## Generated dbt project structure
|
|
@@ -85,7 +85,7 @@ access required.
|
|
|
85
85
|
## The model file
|
|
86
86
|
|
|
87
87
|
A model is one YAML document, `<name>.model2data.yml` — or the same document as JSON. Its format
|
|
88
|
-
is [spec 0.
|
|
88
|
+
is [spec 0.3.0](https://github.com/JB-Analytica/model2data/blob/main/model2data/spec/README.md), with a JSON Schema your editor can autocomplete and
|
|
89
89
|
check against:
|
|
90
90
|
|
|
91
91
|
```yaml
|
|
@@ -283,6 +283,55 @@ days[3].tables["orders"].updated # rows day 3 changed, with their new values
|
|
|
283
283
|
days[3].tables["orders"].state # the table after day 3
|
|
284
284
|
```
|
|
285
285
|
|
|
286
|
+
### Break the data on purpose
|
|
287
|
+
|
|
288
|
+
A dbt test you have never seen fail is a test you hope works. `--defects` puts deliberate,
|
|
289
|
+
counted defects in the generated data, so `dbt build` fails exactly the tests it should — for
|
|
290
|
+
teaching dbt, or for proving a project's tests fire:
|
|
291
|
+
|
|
292
|
+
```bash
|
|
293
|
+
model2data --file examples/ecommerce.model2data.yml --seed 42 --defects training
|
|
294
|
+
cd dbt_ecommerce && dbt build # fails unique, not_null, relationships, accepted_values once each
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
`training` breaks each kind of standard dbt test once; `messy` puts a small share of every defect
|
|
298
|
+
on every table; `clean` (the default) is the data as it always was, byte for byte; `none` also
|
|
299
|
+
ignores the defects a model's tables list, for the clean data of a training model. A table can
|
|
300
|
+
also list its own defects in the model (spec 0.3.0):
|
|
301
|
+
|
|
302
|
+
```yaml
|
|
303
|
+
orders:
|
|
304
|
+
defects:
|
|
305
|
+
- {type: duplicate_keys, count: 3} # fails unique
|
|
306
|
+
- {type: orphan_foreign_keys, column: customer_id, share: 0.02} # fails relationships
|
|
307
|
+
- {type: nulls, column: order_date, count: 5} # fails not_null
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
The types are `duplicate_keys`, `orphan_foreign_keys`, `nulls`, `invalid_values` (fails
|
|
311
|
+
`accepted_values`), `messy_text` (whitespace and casing to clean in staging), and, with `--days`,
|
|
312
|
+
`late_arriving` (rows an incremental model filtering on `updated_at > max(updated_at)` skips) and
|
|
313
|
+
`late_updates` (versions a `strategy: timestamp` snapshot misses) and `overlapping_history` (see
|
|
314
|
+
below). Every run with defects writes
|
|
315
|
+
`defects_report.json` (each defect, the rows it broke, the tests it breaks) and
|
|
316
|
+
`EXPECTED_FAILURES.md` into the project, so a test that does not fire is visible. Defects are applied
|
|
317
|
+
after the clean data, from their own random stream: the same seed gives the same bytes, and a
|
|
318
|
+
defect on one table moves nothing in another. From Python, `model2data.defects.planned_defects`
|
|
319
|
+
says what a run will break, `model2data.output.finish_run` applies it (and builds the histories)
|
|
320
|
+
after `generate_days`, and `write_defects_report` / `write_expected_failures` write the two files.
|
|
321
|
+
|
|
322
|
+
### A source that keeps its history
|
|
323
|
+
|
|
324
|
+
`history: true` in a table's `incremental` (spec 0.3.0) also writes `<table>_history`: every
|
|
325
|
+
version of every row over the generated days, with `valid_from`, `valid_to` (null for the current
|
|
326
|
+
version) and `is_current` — an SCD type 2 source to build snapshots and point-in-time joins
|
|
327
|
+
against. The project tests it with two package-free tests, one current row per key and no
|
|
328
|
+
overlapping versions; the `overlapping_history` defect breaks the second.
|
|
329
|
+
|
|
330
|
+
```yaml
|
|
331
|
+
orders:
|
|
332
|
+
incremental: {new_per_day: 20, update_rate: 0.1, updated_at: updated_at, history: true}
|
|
333
|
+
```
|
|
334
|
+
|
|
286
335
|
## Generated dbt project structure
|
|
287
336
|
|
|
288
337
|
The generated dbt project includes:
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import random
|
|
2
|
-
import re
|
|
3
2
|
import shutil
|
|
4
|
-
from
|
|
3
|
+
from collections.abc import Iterable
|
|
5
4
|
from datetime import date, datetime
|
|
6
5
|
from pathlib import Path
|
|
7
6
|
from typing import Any, Optional
|
|
@@ -13,12 +12,19 @@ from typer.core import TyperGroup
|
|
|
13
12
|
from typer.models import ParameterInfo
|
|
14
13
|
|
|
15
14
|
from model2data.dbt.hint_tests import DEFAULT_TOLERANCE, SEVERITIES
|
|
15
|
+
from model2data.dbt.naming import dbt_names, for_dbt
|
|
16
16
|
from model2data.dbt.project import (
|
|
17
17
|
create_profiles_yml,
|
|
18
18
|
create_project_scaffold,
|
|
19
19
|
create_staging_models,
|
|
20
20
|
)
|
|
21
21
|
from model2data.dbt.tests import generate_dbt_yml, generate_unit_tests
|
|
22
|
+
from model2data.defects import (
|
|
23
|
+
DefectsReport,
|
|
24
|
+
planned_defects,
|
|
25
|
+
write_defects_report,
|
|
26
|
+
write_expected_failures,
|
|
27
|
+
)
|
|
22
28
|
from model2data.generate.core import (
|
|
23
29
|
generate_data_from_dbml,
|
|
24
30
|
get_cyclic_tables,
|
|
@@ -36,8 +42,10 @@ from model2data.generate.faker import (
|
|
|
36
42
|
get_unmapped_columns,
|
|
37
43
|
reset_stats,
|
|
38
44
|
)
|
|
45
|
+
from model2data.generate.history import history_tables
|
|
39
46
|
from model2data.generate.options import TimeProfile
|
|
40
47
|
from model2data.model import (
|
|
48
|
+
DEFECT_PRESETS,
|
|
41
49
|
Issue,
|
|
42
50
|
Model,
|
|
43
51
|
ModelError,
|
|
@@ -47,7 +55,7 @@ from model2data.model import (
|
|
|
47
55
|
to_engine,
|
|
48
56
|
validate,
|
|
49
57
|
)
|
|
50
|
-
from model2data.
|
|
58
|
+
from model2data.output import finish_run
|
|
51
59
|
from model2data.utils import normalize_identifier
|
|
52
60
|
|
|
53
61
|
SUPPORTED_ADAPTERS = ("duckdb", "postgres")
|
|
@@ -155,7 +163,7 @@ app = typer.Typer(
|
|
|
155
163
|
cls=_GenerateByDefault,
|
|
156
164
|
help=(
|
|
157
165
|
"model2data: Generate analytics-ready datasets from a data model.\n\n"
|
|
158
|
-
"Given a model -- a .model2data.yml document (spec 0.
|
|
166
|
+
"Given a model -- a .model2data.yml document (spec 0.3.0), the same as JSON,\n"
|
|
159
167
|
"or a DBML file -- this tool produces:\n"
|
|
160
168
|
"• Synthetic but realistic data\n"
|
|
161
169
|
"• A runnable dbt project scaffold\n"
|
|
@@ -211,37 +219,13 @@ def _model_stem(file: Path) -> str:
|
|
|
211
219
|
return file.stem
|
|
212
220
|
|
|
213
221
|
|
|
214
|
-
def
|
|
215
|
-
"""A table key as the seed and model name it gets in the dbt project.
|
|
216
|
-
|
|
217
|
-
Spec 0.2.0 leaves a name as written (`user accounts`, `raw.orders`) and has
|
|
218
|
-
a consumer normalise it where it reaches a file or a dbt identifier, to
|
|
219
|
-
`[a-z0-9_]`. Underscores are kept as they are, so `_dlt_loads` and
|
|
220
|
-
`stories__kids` stay what they were.
|
|
221
|
-
"""
|
|
222
|
-
cleaned = re.sub(r"[^0-9a-z_]+", "_", key.lower())
|
|
223
|
-
if cleaned.strip("_") == "":
|
|
224
|
-
cleaned = "table"
|
|
225
|
-
elif cleaned[0].isdigit():
|
|
226
|
-
cleaned = f"t_{cleaned}"
|
|
227
|
-
return cleaned
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
def _dbt_names(tables: dict[str, TableDef]) -> dict[str, str]:
|
|
222
|
+
def _dbt_names(tables: Iterable[str]) -> dict[str, str]:
|
|
231
223
|
"""Each table key's dbt identifier, refusing two keys that normalise alike."""
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
typer.echo(
|
|
238
|
-
f"❌ Tables {owners[name]!r} and {key!r} would both be the dbt seed {name!r}. "
|
|
239
|
-
"Rename one of them."
|
|
240
|
-
)
|
|
241
|
-
raise typer.Exit(1)
|
|
242
|
-
owners[name] = key
|
|
243
|
-
names[key] = name
|
|
244
|
-
return names
|
|
224
|
+
try:
|
|
225
|
+
return dbt_names(tables)
|
|
226
|
+
except ValueError as exc:
|
|
227
|
+
typer.echo(f"❌ {exc}")
|
|
228
|
+
raise typer.Exit(1) from None
|
|
245
229
|
|
|
246
230
|
|
|
247
231
|
@app.command(
|
|
@@ -420,6 +404,18 @@ def main(
|
|
|
420
404
|
"--next",
|
|
421
405
|
help="Shorthand for --days 1.",
|
|
422
406
|
),
|
|
407
|
+
defects_preset: Optional[str] = typer.Option(
|
|
408
|
+
None,
|
|
409
|
+
"--defects",
|
|
410
|
+
metavar="PRESET",
|
|
411
|
+
help=(
|
|
412
|
+
"Break the data on purpose: 'clean' (no preset, the default), 'messy' (a mix on\n"
|
|
413
|
+
"every table) or 'training' (each kind of dbt test fails once); a table's own\n"
|
|
414
|
+
"`defects` still apply. 'none' ignores every defect, the tables' own too.\n"
|
|
415
|
+
"Overrides the model's run.defects. Writes defects_report.json and\n"
|
|
416
|
+
"EXPECTED_FAILURES.md into the project."
|
|
417
|
+
),
|
|
418
|
+
),
|
|
423
419
|
days_format: str = typer.Option(
|
|
424
420
|
"batches",
|
|
425
421
|
"--days-format",
|
|
@@ -463,6 +459,14 @@ def main(
|
|
|
463
459
|
tolerance = _given(test_tolerance)
|
|
464
460
|
tolerance = DEFAULT_TOLERANCE if tolerance is None else float(tolerance)
|
|
465
461
|
|
|
462
|
+
preset = _given(defects_preset)
|
|
463
|
+
if preset is not None:
|
|
464
|
+
preset = preset.lower()
|
|
465
|
+
if preset not in DEFECT_PRESETS:
|
|
466
|
+
raise typer.BadParameter(
|
|
467
|
+
f"Choose one of: {', '.join(DEFECT_PRESETS)}.", param_hint="--defects"
|
|
468
|
+
)
|
|
469
|
+
|
|
466
470
|
# -------------------------
|
|
467
471
|
# Read the model (names untouched)
|
|
468
472
|
# -------------------------
|
|
@@ -537,7 +541,11 @@ def main(
|
|
|
537
541
|
if table_seeds:
|
|
538
542
|
rolled = ", ".join(f"{key}={value}" for key, value in sorted(table_seeds.items()))
|
|
539
543
|
typer.echo(f"🎲 Re-rolling with a table seed of its own: {rolled}")
|
|
540
|
-
|
|
544
|
+
names = _dbt_names([*tables, *history_tables(inputs)])
|
|
545
|
+
plan = planned_defects(model, preset, days=days or 0)
|
|
546
|
+
preset = preset or (run.defects if run.defects is not None else None)
|
|
547
|
+
if preset in ("messy", "training") and not plan:
|
|
548
|
+
typer.echo(f"ℹ️ The {preset} preset finds nothing to break in this model: no defects.")
|
|
541
549
|
|
|
542
550
|
project_name = normalize_identifier(_given(name) or model.name or _model_stem(file))
|
|
543
551
|
dest = Path.cwd() / f"dbt_{project_name}"
|
|
@@ -596,22 +604,33 @@ def main(
|
|
|
596
604
|
typer.echo(f"❌ {exc}")
|
|
597
605
|
raise typer.Exit(1) from None
|
|
598
606
|
|
|
607
|
+
# -------------------------
|
|
608
|
+
# Defects: broken on purpose, after the clean data, and reported
|
|
609
|
+
# -------------------------
|
|
610
|
+
if plan:
|
|
611
|
+
typer.echo("🧨 Breaking the data on purpose (defects)...")
|
|
612
|
+
out = finish_run(
|
|
613
|
+
inputs,
|
|
614
|
+
day_results or generated_tables,
|
|
615
|
+
plan,
|
|
616
|
+
seed=seed,
|
|
617
|
+
preset=preset,
|
|
618
|
+
as_of=anchor,
|
|
619
|
+
hint_tests=hint_tests,
|
|
620
|
+
test_tolerance=tolerance,
|
|
621
|
+
names=names,
|
|
622
|
+
)
|
|
623
|
+
report = out.report
|
|
624
|
+
generated_tables = out.frames
|
|
625
|
+
if day_results:
|
|
626
|
+
day_results = out.days
|
|
627
|
+
|
|
599
628
|
# -------------------------
|
|
600
629
|
# dbt names: a table key becomes a [a-z0-9_] seed and model name here,
|
|
601
630
|
# once, so the CSV's stem, `stg_<name>` and every `ref()` agree.
|
|
602
631
|
# -------------------------
|
|
603
|
-
dbt_tables =
|
|
604
|
-
|
|
605
|
-
}
|
|
606
|
-
dbt_refs = [
|
|
607
|
-
{
|
|
608
|
-
**ref,
|
|
609
|
-
"source_table": dbt_names.get(ref["source_table"], ref["source_table"]),
|
|
610
|
-
"target_table": dbt_names.get(ref["target_table"], ref["target_table"]),
|
|
611
|
-
}
|
|
612
|
-
for ref in refs
|
|
613
|
-
]
|
|
614
|
-
dbt_frames = {dbt_names.get(key, key): df for key, df in generated_tables.items()}
|
|
632
|
+
dbt_tables, dbt_refs = for_dbt(out.tables, refs, names)
|
|
633
|
+
dbt_frames = {names.get(key, key): df for key, df in generated_tables.items()}
|
|
615
634
|
|
|
616
635
|
# -------------------------
|
|
617
636
|
# Write dbt seeds (normalized names)
|
|
@@ -623,7 +642,7 @@ def main(
|
|
|
623
642
|
|
|
624
643
|
if day_results and days_format != "final":
|
|
625
644
|
write = write_batches if days_format == "batches" else write_changelog
|
|
626
|
-
write(dest, day_results,
|
|
645
|
+
write(dest, day_results, names)
|
|
627
646
|
|
|
628
647
|
# -------------------------
|
|
629
648
|
# Build dbt assets
|
|
@@ -651,6 +670,10 @@ def main(
|
|
|
651
670
|
# Keep the original model file for reference
|
|
652
671
|
shutil.copy(file, dest / file.name)
|
|
653
672
|
|
|
673
|
+
if report is not None:
|
|
674
|
+
write_defects_report(report, dest)
|
|
675
|
+
write_expected_failures(report, dest)
|
|
676
|
+
|
|
654
677
|
# -------------------------
|
|
655
678
|
# Summary
|
|
656
679
|
# -------------------------
|
|
@@ -698,6 +721,9 @@ def main(
|
|
|
698
721
|
for label in duplicate_unique:
|
|
699
722
|
typer.echo(f" - {label}")
|
|
700
723
|
|
|
724
|
+
if report is not None:
|
|
725
|
+
_print_defects(report)
|
|
726
|
+
|
|
701
727
|
# -------------------------
|
|
702
728
|
# Done
|
|
703
729
|
# -------------------------
|
|
@@ -707,6 +733,29 @@ def main(
|
|
|
707
733
|
typer.echo(" dbt build # loads the seeds, builds the models, runs every test")
|
|
708
734
|
|
|
709
735
|
|
|
736
|
+
def _spec_of(file: Path) -> str:
|
|
737
|
+
"""The spec a conforming model is written against: 0.3.0, or 0.2.0 (DBML included)."""
|
|
738
|
+
return "0.3.0" if str(load(file).version).startswith("0.3") else "0.2.0"
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def _print_defects(report: DefectsReport) -> None:
|
|
742
|
+
typer.echo("\n🧨 Defects (defects_report.json, EXPECTED_FAILURES.md)")
|
|
743
|
+
for defect in report.defects:
|
|
744
|
+
column = defect.column if not isinstance(defect.column, list) else ", ".join(defect.column)
|
|
745
|
+
on = f".{column}" if column else ""
|
|
746
|
+
typer.echo(
|
|
747
|
+
f" {defect.table}{on}: {defect.defect}, {defect.applied} "
|
|
748
|
+
f"row{'' if defect.applied == 1 else 's'}"
|
|
749
|
+
)
|
|
750
|
+
if defect.note and not defect.applied:
|
|
751
|
+
typer.echo(f" ⚠️ {defect.note}")
|
|
752
|
+
count = len(report.expected_failures)
|
|
753
|
+
typer.echo(f" dbt build should fail {count} test{'' if count == 1 else 's'}:")
|
|
754
|
+
for failure in report.expected_failures:
|
|
755
|
+
warn = " (warn)" if failure.severity == "warn" else ""
|
|
756
|
+
typer.echo(f" - {failure.test}{warn}")
|
|
757
|
+
|
|
758
|
+
|
|
710
759
|
@app.command("validate")
|
|
711
760
|
def validate_command(
|
|
712
761
|
file: Path = typer.Argument( # noqa: B008
|
|
@@ -718,7 +767,7 @@ def validate_command(
|
|
|
718
767
|
help="The model: a .model2data.yml / .yaml / .json document, or a .dbml file.",
|
|
719
768
|
),
|
|
720
769
|
):
|
|
721
|
-
"""Check that a model conforms to spec 0.2.
|
|
770
|
+
"""Check that a model conforms to spec 0.3.0 (or 0.2.x), printing every issue with its path.
|
|
722
771
|
|
|
723
772
|
Exits 1 when there is an error; warnings are printed, and the model conforms.
|
|
724
773
|
"""
|
|
@@ -726,7 +775,7 @@ def validate_command(
|
|
|
726
775
|
_print_issues(file, issues)
|
|
727
776
|
if any(issue.is_error for issue in issues):
|
|
728
777
|
raise typer.Exit(1)
|
|
729
|
-
typer.echo(f"✅ {file.name} conforms to spec
|
|
778
|
+
typer.echo(f"✅ {file.name} conforms to spec {_spec_of(file)}.")
|
|
730
779
|
|
|
731
780
|
|
|
732
781
|
@app.command("convert")
|
|
@@ -748,7 +797,8 @@ def convert_command(
|
|
|
748
797
|
),
|
|
749
798
|
force: bool = typer.Option(False, "--force", help="Overwrite the output file if it exists."),
|
|
750
799
|
):
|
|
751
|
-
"""Convert a model -- DBML, typically -- to a spec 0.2.0
|
|
800
|
+
"""Convert a model -- DBML, typically -- to a .model2data.yml document (spec 0.2.0, or
|
|
801
|
+
0.3.0 when it has defects)."""
|
|
752
802
|
try:
|
|
753
803
|
model = load(file)
|
|
754
804
|
except ModelError as error:
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""A model's table keys as the seed and model names they get in a dbt project.
|
|
2
|
+
|
|
3
|
+
The CLI and `model2data.defects` both name tables this way, so a test name the
|
|
4
|
+
defects report predicts is the one dbt gives the test in the generated project.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
from collections.abc import Iterable, Mapping
|
|
11
|
+
from dataclasses import replace
|
|
12
|
+
|
|
13
|
+
from model2data.parse.dbml import TableDef
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def dbt_identifier(key: str) -> str:
|
|
17
|
+
"""A table key as the seed and model name it gets in the dbt project.
|
|
18
|
+
|
|
19
|
+
Spec 0.2.0 leaves a name as written (`user accounts`, `raw.orders`) and has
|
|
20
|
+
a consumer normalise it where it reaches a file or a dbt identifier, to
|
|
21
|
+
`[a-z0-9_]`. Underscores are kept as they are, so `_dlt_loads` and
|
|
22
|
+
`stories__kids` stay what they were.
|
|
23
|
+
"""
|
|
24
|
+
cleaned = re.sub(r"[^0-9a-z_]+", "_", key.lower())
|
|
25
|
+
if cleaned.strip("_") == "":
|
|
26
|
+
cleaned = "table"
|
|
27
|
+
elif cleaned[0].isdigit():
|
|
28
|
+
cleaned = f"t_{cleaned}"
|
|
29
|
+
return cleaned
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def dbt_names(keys: Iterable[str]) -> dict[str, str]:
|
|
33
|
+
"""Each table key's dbt identifier; ValueError when two keys normalise alike."""
|
|
34
|
+
names: dict[str, str] = {}
|
|
35
|
+
owners: dict[str, str] = {}
|
|
36
|
+
for key in keys:
|
|
37
|
+
name = dbt_identifier(key)
|
|
38
|
+
if name in owners:
|
|
39
|
+
raise ValueError(
|
|
40
|
+
f"Tables {owners[name]!r} and {key!r} would both be the dbt seed {name!r}. "
|
|
41
|
+
"Rename one of them."
|
|
42
|
+
)
|
|
43
|
+
owners[name] = key
|
|
44
|
+
names[key] = name
|
|
45
|
+
return names
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def for_dbt(
|
|
49
|
+
tables: Mapping[str, TableDef], refs: list[dict], names: Mapping[str, str]
|
|
50
|
+
) -> tuple[dict[str, TableDef], list[dict]]:
|
|
51
|
+
"""`(tables, refs)` renamed to their dbt names, as `generate_dbt_yml` takes them."""
|
|
52
|
+
dbt_tables = {names[key]: replace(table, name=names[key]) for key, table in tables.items()}
|
|
53
|
+
dbt_refs = [
|
|
54
|
+
{
|
|
55
|
+
**ref,
|
|
56
|
+
"source_table": names.get(ref["source_table"], ref["source_table"]),
|
|
57
|
+
"target_table": names.get(ref["target_table"], ref["target_table"]),
|
|
58
|
+
}
|
|
59
|
+
for ref in refs
|
|
60
|
+
]
|
|
61
|
+
return dbt_tables, dbt_refs
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{#-
|
|
2
|
+
Generic tests of a history table (`incremental.history`): one row per version
|
|
3
|
+
of each key, valid from `valid_from` until `valid_to` (null for the current
|
|
4
|
+
version). Written only into a project that has a history table. Plain SQL, no
|
|
5
|
+
dbt package. Each test returns the keys (or versions) that break it.
|
|
6
|
+
-#}
|
|
7
|
+
|
|
8
|
+
{% test model2data_one_current_row(model, key, current) %}
|
|
9
|
+
{%- set keys = [] -%}
|
|
10
|
+
{%- for column in key -%}{%- do keys.append(adapter.quote(column)) -%}{%- endfor %}
|
|
11
|
+
select {{ keys | join(', ') }},
|
|
12
|
+
sum(case when {{ adapter.quote(current) }} then 1 else 0 end) as current_rows
|
|
13
|
+
from {{ model }}
|
|
14
|
+
group by {{ keys | join(', ') }}
|
|
15
|
+
having sum(case when {{ adapter.quote(current) }} then 1 else 0 end) <> 1
|
|
16
|
+
{% endtest %}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
{#-
|
|
20
|
+
A version overlaps the next one of its key when it is still valid when the
|
|
21
|
+
next begins: no `valid_to`, or one after the next version's `valid_from`.
|
|
22
|
+
-#}
|
|
23
|
+
{% test model2data_no_overlapping_ranges(model, key, valid_from, valid_to) %}
|
|
24
|
+
{%- set keys = [] -%}
|
|
25
|
+
{%- for column in key -%}{%- do keys.append(adapter.quote(column)) -%}{%- endfor %}
|
|
26
|
+
select *
|
|
27
|
+
from (
|
|
28
|
+
select {{ keys | join(', ') }},
|
|
29
|
+
{{ adapter.quote(valid_from) }} as version_from,
|
|
30
|
+
{{ adapter.quote(valid_to) }} as version_to,
|
|
31
|
+
lead({{ adapter.quote(valid_from) }}) over (
|
|
32
|
+
partition by {{ keys | join(', ') }} order by {{ adapter.quote(valid_from) }}
|
|
33
|
+
) as next_from
|
|
34
|
+
from {{ model }}
|
|
35
|
+
) as versions
|
|
36
|
+
where next_from is not null
|
|
37
|
+
and (version_to is null or version_to > next_from)
|
|
38
|
+
{% endtest %}
|