model2data 1.8.0__tar.gz → 1.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. {model2data-1.8.0/model2data.egg-info → model2data-1.9.0}/PKG-INFO +51 -2
  2. {model2data-1.8.0 → model2data-1.9.0}/README.md +50 -1
  3. {model2data-1.8.0 → model2data-1.9.0}/README_PYPI.md +50 -1
  4. {model2data-1.8.0 → model2data-1.9.0}/model2data/cli.py +101 -51
  5. model2data-1.9.0/model2data/dbt/naming.py +61 -0
  6. model2data-1.9.0/model2data/dbt/templates/history_macros/model2data_history_tests.sql +38 -0
  7. {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/tests.py +253 -55
  8. model2data-1.9.0/model2data/defects/__init__.py +59 -0
  9. model2data-1.9.0/model2data/defects/apply.py +1064 -0
  10. model2data-1.9.0/model2data/defects/checks.py +179 -0
  11. model2data-1.9.0/model2data/defects/presets.py +311 -0
  12. model2data-1.9.0/model2data/defects/report.py +245 -0
  13. {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/days.py +9 -4
  14. model2data-1.9.0/model2data/generate/history.py +152 -0
  15. {model2data-1.8.0 → model2data-1.9.0}/model2data/model/__init__.py +7 -1
  16. {model2data-1.8.0 → model2data-1.9.0}/model2data/model/document.py +50 -1
  17. {model2data-1.8.0 → model2data-1.9.0}/model2data/model/dump.py +15 -3
  18. {model2data-1.8.0 → model2data-1.9.0}/model2data/model/types.py +39 -1
  19. {model2data-1.8.0 → model2data-1.9.0}/model2data/model/validate.py +302 -7
  20. model2data-1.9.0/model2data/output.py +113 -0
  21. {model2data-1.8.0 → model2data-1.9.0}/model2data/spec/PUBLISHING.md +13 -9
  22. {model2data-1.8.0 → model2data-1.9.0}/model2data/spec/README.md +92 -3
  23. {model2data-1.8.0 → model2data-1.9.0}/model2data/spec/model.schema.json +59 -5
  24. {model2data-1.8.0 → model2data-1.9.0/model2data.egg-info}/PKG-INFO +51 -2
  25. {model2data-1.8.0 → model2data-1.9.0}/model2data.egg-info/SOURCES.txt +15 -0
  26. {model2data-1.8.0 → model2data-1.9.0}/pyproject.toml +1 -1
  27. {model2data-1.8.0 → model2data-1.9.0}/tests/test_dbt_tests.py +44 -0
  28. model2data-1.9.0/tests/test_defects.py +772 -0
  29. model2data-1.9.0/tests/test_defects_dbt.py +238 -0
  30. model2data-1.9.0/tests/test_defects_replay.py +138 -0
  31. model2data-1.9.0/tests/test_defects_report.py +370 -0
  32. model2data-1.9.0/tests/test_defects_review.py +302 -0
  33. {model2data-1.8.0 → model2data-1.9.0}/tests/test_hint_tests.py +12 -1
  34. model2data-1.9.0/tests/test_history.py +254 -0
  35. {model2data-1.8.0 → model2data-1.9.0}/tests/test_model_edges.py +2 -1
  36. model2data-1.9.0/tests/test_model_messages.py +603 -0
  37. model2data-1.8.0/tests/test_model_messages.py +0 -250
  38. {model2data-1.8.0 → model2data-1.9.0}/LICENSE +0 -0
  39. {model2data-1.8.0 → model2data-1.9.0}/model2data/__init__.py +0 -0
  40. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/__init__.py +0 -0
  41. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/LICENSE +0 -0
  42. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/VENDORED.md +0 -0
  43. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/__init__.py +0 -0
  44. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/__init__.py +0 -0
  45. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/base.py +0 -0
  46. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/column.py +0 -0
  47. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/enum.py +0 -0
  48. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/expression.py +0 -0
  49. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/index.py +0 -0
  50. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/note.py +0 -0
  51. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/project.py +0 -0
  52. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/reference.py +0 -0
  53. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/sticky_note.py +0 -0
  54. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/table.py +0 -0
  55. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/_classes/table_group.py +0 -0
  56. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/classes/__init__.py +0 -0
  57. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/constants.py +0 -0
  58. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/database.py +0 -0
  59. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/__init__.py +0 -0
  60. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/column.py +0 -0
  61. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/common.py +0 -0
  62. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/enum.py +0 -0
  63. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/generic.py +0 -0
  64. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/index.py +0 -0
  65. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/project.py +0 -0
  66. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/records.py +0 -0
  67. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/reference.py +0 -0
  68. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/sticky_note.py +0 -0
  69. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/table.py +0 -0
  70. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/table_group.py +0 -0
  71. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/definitions/table_partial.py +0 -0
  72. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/exceptions.py +0 -0
  73. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/parser/__init__.py +0 -0
  74. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/parser/blueprints.py +0 -0
  75. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/parser/parser.py +0 -0
  76. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/__init__.py +0 -0
  77. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/base.py +0 -0
  78. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/__init__.py +0 -0
  79. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/__init__.py +0 -0
  80. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/column.py +0 -0
  81. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/enum.py +0 -0
  82. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/expression.py +0 -0
  83. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/index.py +0 -0
  84. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/note.py +0 -0
  85. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/project.py +0 -0
  86. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/reference.py +0 -0
  87. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/renderer.py +0 -0
  88. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/sticky_note.py +0 -0
  89. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/table.py +0 -0
  90. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/table_group.py +0 -0
  91. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/dbml/default/utils.py +0 -0
  92. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/__init__.py +0 -0
  93. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/__init__.py +0 -0
  94. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/column.py +0 -0
  95. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/enum.py +0 -0
  96. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/expression.py +0 -0
  97. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/index.py +0 -0
  98. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/note.py +0 -0
  99. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/reference.py +0 -0
  100. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/renderer.py +0 -0
  101. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/table.py +0 -0
  102. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/renderer/sql/default/utils.py +0 -0
  103. {model2data-1.8.0 → model2data-1.9.0}/model2data/_vendor/pydbml/tools.py +0 -0
  104. {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/__init__.py +0 -0
  105. {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/hint_tests.py +0 -0
  106. {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/project.py +0 -0
  107. {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/templates/dbt_project.yml.jinja +0 -0
  108. {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/templates/hint_macros/model2data_hint_tests.sql +0 -0
  109. {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/templates/macros/generate_schema_name.sql +0 -0
  110. {model2data-1.8.0 → model2data-1.9.0}/model2data/dbt/templates/profiles.yml.jinja +0 -0
  111. {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/__init__.py +0 -0
  112. {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/core.py +0 -0
  113. {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/faker.py +0 -0
  114. {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/hints.py +0 -0
  115. {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/kinds.py +0 -0
  116. {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/options.py +0 -0
  117. {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/relationships.py +0 -0
  118. {model2data-1.8.0 → model2data-1.9.0}/model2data/generate/timeline.py +0 -0
  119. {model2data-1.8.0 → model2data-1.9.0}/model2data/model/_yaml.py +0 -0
  120. {model2data-1.8.0 → model2data-1.9.0}/model2data/model/dbml.py +0 -0
  121. {model2data-1.8.0 → model2data-1.9.0}/model2data/model/engine.py +0 -0
  122. {model2data-1.8.0 → model2data-1.9.0}/model2data/model/errors.py +0 -0
  123. {model2data-1.8.0 → model2data-1.9.0}/model2data/model/reader.py +0 -0
  124. {model2data-1.8.0 → model2data-1.9.0}/model2data/parse/__init__.py +0 -0
  125. {model2data-1.8.0 → model2data-1.9.0}/model2data/parse/dbml.py +0 -0
  126. {model2data-1.8.0 → model2data-1.9.0}/model2data/spec/examples/coffee_webshop.model2data.yml +0 -0
  127. {model2data-1.8.0 → model2data-1.9.0}/model2data/utils.py +0 -0
  128. {model2data-1.8.0 → model2data-1.9.0}/model2data.egg-info/dependency_links.txt +0 -0
  129. {model2data-1.8.0 → model2data-1.9.0}/model2data.egg-info/entry_points.txt +0 -0
  130. {model2data-1.8.0 → model2data-1.9.0}/model2data.egg-info/requires.txt +0 -0
  131. {model2data-1.8.0 → model2data-1.9.0}/model2data.egg-info/top_level.txt +0 -0
  132. {model2data-1.8.0 → model2data-1.9.0}/setup.cfg +0 -0
  133. {model2data-1.8.0 → model2data-1.9.0}/tests/test_as_of_anchor.py +0 -0
  134. {model2data-1.8.0 → model2data-1.9.0}/tests/test_cli.py +0 -0
  135. {model2data-1.8.0 → model2data-1.9.0}/tests/test_cli_model.py +0 -0
  136. {model2data-1.8.0 → model2data-1.9.0}/tests/test_column_time_hints.py +0 -0
  137. {model2data-1.8.0 → model2data-1.9.0}/tests/test_coverage_gaps.py +0 -0
  138. {model2data-1.8.0 → model2data-1.9.0}/tests/test_days.py +0 -0
  139. {model2data-1.8.0 → model2data-1.9.0}/tests/test_dbml_parser.py +0 -0
  140. {model2data-1.8.0 → model2data-1.9.0}/tests/test_dbml_parser_fuzz.py +0 -0
  141. {model2data-1.8.0 → model2data-1.9.0}/tests/test_dbt_integration.py +0 -0
  142. {model2data-1.8.0 → model2data-1.9.0}/tests/test_dbt_naming.py +0 -0
  143. {model2data-1.8.0 → model2data-1.9.0}/tests/test_dbt_project.py +0 -0
  144. {model2data-1.8.0 → model2data-1.9.0}/tests/test_distributions.py +0 -0
  145. {model2data-1.8.0 → model2data-1.9.0}/tests/test_examples_model.py +0 -0
  146. {model2data-1.8.0 → model2data-1.9.0}/tests/test_faker_name_inference.py +0 -0
  147. {model2data-1.8.0 → model2data-1.9.0}/tests/test_generation.py +0 -0
  148. {model2data-1.8.0 → model2data-1.9.0}/tests/test_lone_country.py +0 -0
  149. {model2data-1.8.0 → model2data-1.9.0}/tests/test_model.py +0 -0
  150. {model2data-1.8.0 → model2data-1.9.0}/tests/test_model_dbml.py +0 -0
  151. {model2data-1.8.0 → model2data-1.9.0}/tests/test_model_vocabulary.py +0 -0
  152. {model2data-1.8.0 → model2data-1.9.0}/tests/test_options.py +0 -0
  153. {model2data-1.8.0 → model2data-1.9.0}/tests/test_release_stress.py +0 -0
  154. {model2data-1.8.0 → model2data-1.9.0}/tests/test_reproducibility.py +0 -0
  155. {model2data-1.8.0 → model2data-1.9.0}/tests/test_row_identity.py +0 -0
  156. {model2data-1.8.0 → model2data-1.9.0}/tests/test_shaping.py +0 -0
  157. {model2data-1.8.0 → model2data-1.9.0}/tests/test_spec_alignment.py +0 -0
  158. {model2data-1.8.0 → model2data-1.9.0}/tests/test_spec_schema.py +0 -0
  159. {model2data-1.8.0 → model2data-1.9.0}/tests/test_table_seeds.py +0 -0
  160. {model2data-1.8.0 → model2data-1.9.0}/tests/test_timeline.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: model2data
3
- Version: 1.8.0
3
+ Version: 1.9.0
4
4
  Summary: Generate analytics-ready datasets from DBML models
5
5
  Author: JB Analytica
6
6
  License-Expression: MIT
@@ -132,7 +132,7 @@ access required.
132
132
  ## The model file
133
133
 
134
134
  A model is one YAML document, `<name>.model2data.yml` — or the same document as JSON. Its format
135
- is [spec 0.2.0](https://github.com/JB-Analytica/model2data/blob/main/model2data/spec/README.md), with a JSON Schema your editor can autocomplete and
135
+ is [spec 0.3.0](https://github.com/JB-Analytica/model2data/blob/main/model2data/spec/README.md), with a JSON Schema your editor can autocomplete and
136
136
  check against:
137
137
 
138
138
  ```yaml
@@ -330,6 +330,55 @@ days[3].tables["orders"].updated # rows day 3 changed, with their new values
330
330
  days[3].tables["orders"].state # the table after day 3
331
331
  ```
332
332
 
333
+ ### Break the data on purpose
334
+
335
+ A dbt test you have never seen fail is a test you hope works. `--defects` puts deliberate,
336
+ counted defects in the generated data, so `dbt build` fails exactly the tests it should — for
337
+ teaching dbt, or for proving a project's tests fire:
338
+
339
+ ```bash
340
+ model2data --file examples/ecommerce.model2data.yml --seed 42 --defects training
341
+ cd dbt_ecommerce && dbt build # fails unique, not_null, relationships, accepted_values once each
342
+ ```
343
+
344
+ `training` breaks each kind of standard dbt test once; `messy` puts a small share of every defect
345
+ on every table; `clean` (the default) is the data as it always was, byte for byte; `none` also
346
+ ignores the defects a model's tables list, for the clean data of a training model. A table can
347
+ also list its own defects in the model (spec 0.3.0):
348
+
349
+ ```yaml
350
+ orders:
351
+ defects:
352
+ - {type: duplicate_keys, count: 3} # fails unique
353
+ - {type: orphan_foreign_keys, column: customer_id, share: 0.02} # fails relationships
354
+ - {type: nulls, column: order_date, count: 5} # fails not_null
355
+ ```
356
+
357
+ The types are `duplicate_keys`, `orphan_foreign_keys`, `nulls`, `invalid_values` (fails
358
+ `accepted_values`), `messy_text` (whitespace and casing to clean in staging), and, with `--days`,
359
+ `late_arriving` (rows an incremental model filtering on `updated_at > max(updated_at)` skips) and
360
+ `late_updates` (versions a `strategy: timestamp` snapshot misses) and `overlapping_history` (see
361
+ below). Every run with defects writes
362
+ `defects_report.json` (each defect, the rows it broke, the tests it breaks) and
363
+ `EXPECTED_FAILURES.md` into the project, so a test that does not fire is visible. Defects are applied
364
+ after the clean data, from their own random stream: the same seed gives the same bytes, and a
365
+ defect on one table moves nothing in another. From Python, `model2data.defects.planned_defects`
366
+ says what a run will break, `model2data.output.finish_run` applies it (and builds the histories)
367
+ after `generate_days`, and `write_defects_report` / `write_expected_failures` write the two files.
368
+
369
+ ### A source that keeps its history
370
+
371
+ `history: true` in a table's `incremental` (spec 0.3.0) also writes `<table>_history`: every
372
+ version of every row over the generated days, with `valid_from`, `valid_to` (null for the current
373
+ version) and `is_current` — an SCD type 2 source to build snapshots and point-in-time joins
374
+ against. The project tests it with two package-free tests, one current row per key and no
375
+ overlapping versions; the `overlapping_history` defect breaks the second.
376
+
377
+ ```yaml
378
+ orders:
379
+ incremental: {new_per_day: 20, update_rate: 0.1, updated_at: updated_at, history: true}
380
+ ```
381
+
333
382
  ## Generated dbt project structure
334
383
 
335
384
  The generated dbt project includes:
@@ -123,7 +123,7 @@ flowchart LR
123
123
  ## The model file
124
124
 
125
125
  A model is one YAML document, `<name>.model2data.yml` — or the same document as JSON. Its
126
- format is [spec 0.2.0](model2data/spec/README.md), with a JSON Schema
126
+ format is [spec 0.3.0](model2data/spec/README.md), with a JSON Schema
127
127
  ([`model.schema.json`](model2data/spec/model.schema.json)) your editor can autocomplete and check
128
128
  against:
129
129
 
@@ -473,6 +473,55 @@ they describe intent and should not break a first `dbt build` on real data); the
473
473
  same YAML as before. To call the mapping from Python, `model2data.dbt.hint_tests.hint_tests_for(model)`
474
474
  returns the tests as data.
475
475
 
476
+ ### Break the data on purpose
477
+
478
+ A dbt test you have never seen fail is a test you hope works. `--defects` puts deliberate,
479
+ counted defects in the generated data, so `dbt build` fails exactly the tests it should — for
480
+ teaching dbt, or for proving a project's tests fire:
481
+
482
+ ```bash
483
+ model2data --file examples/ecommerce.model2data.yml --seed 42 --defects training
484
+ cd dbt_ecommerce && dbt build # fails unique, not_null, relationships, accepted_values once each
485
+ ```
486
+
487
+ `training` breaks each kind of standard dbt test once; `messy` puts a small share of every defect
488
+ on every table; `clean` (the default) is the data as it always was, byte for byte; `none` also
489
+ ignores the defects a model's tables list, for the clean data of a training model. A table can
490
+ also list its own defects in the model (spec 0.3.0):
491
+
492
+ ```yaml
493
+ orders:
494
+ defects:
495
+ - {type: duplicate_keys, count: 3} # fails unique
496
+ - {type: orphan_foreign_keys, column: customer_id, share: 0.02} # fails relationships
497
+ - {type: nulls, column: order_date, count: 5} # fails not_null
498
+ ```
499
+
500
+ The types are `duplicate_keys`, `orphan_foreign_keys`, `nulls`, `invalid_values` (fails
501
+ `accepted_values`), `messy_text` (whitespace and casing to clean in staging), and, with `--days`,
502
+ `late_arriving` (rows an incremental model filtering on `updated_at > max(updated_at)` skips) and
503
+ `late_updates` (versions a `strategy: timestamp` snapshot misses) and `overlapping_history` (see
504
+ below). Every run with defects writes
505
+ `defects_report.json` (each defect, the rows it broke, the tests it breaks) and
506
+ `EXPECTED_FAILURES.md` into the project, so a test that does not fire is visible. Defects are applied
507
+ after the clean data, from their own random stream: the same seed gives the same bytes, and a
508
+ defect on one table moves nothing in another. From Python, `model2data.defects.planned_defects`
509
+ says what a run will break, `model2data.output.finish_run` applies it (and builds the histories)
510
+ after `generate_days`, and `write_defects_report` / `write_expected_failures` write the two files.
511
+
512
+ ### A source that keeps its history
513
+
514
+ `history: true` in a table's `incremental` (spec 0.3.0) also writes `<table>_history`: every
515
+ version of every row over the generated days, with `valid_from`, `valid_to` (null for the current
516
+ version) and `is_current` — an SCD type 2 source to build snapshots and point-in-time joins
517
+ against. The project tests it with two package-free tests, one current row per key and no
518
+ overlapping versions; the `overlapping_history` defect breaks the second.
519
+
520
+ ```yaml
521
+ orders:
522
+ incremental: {new_per_day: 20, update_rate: 0.1, updated_at: updated_at, history: true}
523
+ ```
524
+
476
525
  ---
477
526
 
478
527
  ## Generated dbt project structure
@@ -85,7 +85,7 @@ access required.
85
85
  ## The model file
86
86
 
87
87
  A model is one YAML document, `<name>.model2data.yml` — or the same document as JSON. Its format
88
- is [spec 0.2.0](https://github.com/JB-Analytica/model2data/blob/main/model2data/spec/README.md), with a JSON Schema your editor can autocomplete and
88
+ is [spec 0.3.0](https://github.com/JB-Analytica/model2data/blob/main/model2data/spec/README.md), with a JSON Schema your editor can autocomplete and
89
89
  check against:
90
90
 
91
91
  ```yaml
@@ -283,6 +283,55 @@ days[3].tables["orders"].updated # rows day 3 changed, with their new values
283
283
  days[3].tables["orders"].state # the table after day 3
284
284
  ```
285
285
 
286
+ ### Break the data on purpose
287
+
288
+ A dbt test you have never seen fail is a test you hope works. `--defects` puts deliberate,
289
+ counted defects in the generated data, so `dbt build` fails exactly the tests it should — for
290
+ teaching dbt, or for proving a project's tests fire:
291
+
292
+ ```bash
293
+ model2data --file examples/ecommerce.model2data.yml --seed 42 --defects training
294
+ cd dbt_ecommerce && dbt build # fails unique, not_null, relationships, accepted_values once each
295
+ ```
296
+
297
+ `training` breaks each kind of standard dbt test once; `messy` puts a small share of every defect
298
+ on every table; `clean` (the default) is the data as it always was, byte for byte; `none` also
299
+ ignores the defects a model's tables list, for the clean data of a training model. A table can
300
+ also list its own defects in the model (spec 0.3.0):
301
+
302
+ ```yaml
303
+ orders:
304
+ defects:
305
+ - {type: duplicate_keys, count: 3} # fails unique
306
+ - {type: orphan_foreign_keys, column: customer_id, share: 0.02} # fails relationships
307
+ - {type: nulls, column: order_date, count: 5} # fails not_null
308
+ ```
309
+
310
+ The types are `duplicate_keys`, `orphan_foreign_keys`, `nulls`, `invalid_values` (fails
311
+ `accepted_values`), `messy_text` (whitespace and casing to clean in staging), and, with `--days`,
312
+ `late_arriving` (rows an incremental model filtering on `updated_at > max(updated_at)` skips) and
313
+ `late_updates` (versions a `strategy: timestamp` snapshot misses) and `overlapping_history` (see
314
+ below). Every run with defects writes
315
+ `defects_report.json` (each defect, the rows it broke, the tests it breaks) and
316
+ `EXPECTED_FAILURES.md` into the project, so a test that does not fire is visible. Defects are applied
317
+ after the clean data, from their own random stream: the same seed gives the same bytes, and a
318
+ defect on one table moves nothing in another. From Python, `model2data.defects.planned_defects`
319
+ says what a run will break, `model2data.output.finish_run` applies it (and builds the histories)
320
+ after `generate_days`, and `write_defects_report` / `write_expected_failures` write the two files.
321
+
322
+ ### A source that keeps its history
323
+
324
+ `history: true` in a table's `incremental` (spec 0.3.0) also writes `<table>_history`: every
325
+ version of every row over the generated days, with `valid_from`, `valid_to` (null for the current
326
+ version) and `is_current` — an SCD type 2 source to build snapshots and point-in-time joins
327
+ against. The project tests it with two package-free tests, one current row per key and no
328
+ overlapping versions; the `overlapping_history` defect breaks the second.
329
+
330
+ ```yaml
331
+ orders:
332
+ incremental: {new_per_day: 20, update_rate: 0.1, updated_at: updated_at, history: true}
333
+ ```
334
+
286
335
  ## Generated dbt project structure
287
336
 
288
337
  The generated dbt project includes:
@@ -1,7 +1,6 @@
1
1
  import random
2
- import re
3
2
  import shutil
4
- from dataclasses import replace
3
+ from collections.abc import Iterable
5
4
  from datetime import date, datetime
6
5
  from pathlib import Path
7
6
  from typing import Any, Optional
@@ -13,12 +12,19 @@ from typer.core import TyperGroup
13
12
  from typer.models import ParameterInfo
14
13
 
15
14
  from model2data.dbt.hint_tests import DEFAULT_TOLERANCE, SEVERITIES
15
+ from model2data.dbt.naming import dbt_names, for_dbt
16
16
  from model2data.dbt.project import (
17
17
  create_profiles_yml,
18
18
  create_project_scaffold,
19
19
  create_staging_models,
20
20
  )
21
21
  from model2data.dbt.tests import generate_dbt_yml, generate_unit_tests
22
+ from model2data.defects import (
23
+ DefectsReport,
24
+ planned_defects,
25
+ write_defects_report,
26
+ write_expected_failures,
27
+ )
22
28
  from model2data.generate.core import (
23
29
  generate_data_from_dbml,
24
30
  get_cyclic_tables,
@@ -36,8 +42,10 @@ from model2data.generate.faker import (
36
42
  get_unmapped_columns,
37
43
  reset_stats,
38
44
  )
45
+ from model2data.generate.history import history_tables
39
46
  from model2data.generate.options import TimeProfile
40
47
  from model2data.model import (
48
+ DEFECT_PRESETS,
41
49
  Issue,
42
50
  Model,
43
51
  ModelError,
@@ -47,7 +55,7 @@ from model2data.model import (
47
55
  to_engine,
48
56
  validate,
49
57
  )
50
- from model2data.parse.dbml import TableDef
58
+ from model2data.output import finish_run
51
59
  from model2data.utils import normalize_identifier
52
60
 
53
61
  SUPPORTED_ADAPTERS = ("duckdb", "postgres")
@@ -155,7 +163,7 @@ app = typer.Typer(
155
163
  cls=_GenerateByDefault,
156
164
  help=(
157
165
  "model2data: Generate analytics-ready datasets from a data model.\n\n"
158
- "Given a model -- a .model2data.yml document (spec 0.2.0), the same as JSON,\n"
166
+ "Given a model -- a .model2data.yml document (spec 0.3.0), the same as JSON,\n"
159
167
  "or a DBML file -- this tool produces:\n"
160
168
  "• Synthetic but realistic data\n"
161
169
  "• A runnable dbt project scaffold\n"
@@ -211,37 +219,13 @@ def _model_stem(file: Path) -> str:
211
219
  return file.stem
212
220
 
213
221
 
214
- def _dbt_identifier(key: str) -> str:
215
- """A table key as the seed and model name it gets in the dbt project.
216
-
217
- Spec 0.2.0 leaves a name as written (`user accounts`, `raw.orders`) and has
218
- a consumer normalise it where it reaches a file or a dbt identifier, to
219
- `[a-z0-9_]`. Underscores are kept as they are, so `_dlt_loads` and
220
- `stories__kids` stay what they were.
221
- """
222
- cleaned = re.sub(r"[^0-9a-z_]+", "_", key.lower())
223
- if cleaned.strip("_") == "":
224
- cleaned = "table"
225
- elif cleaned[0].isdigit():
226
- cleaned = f"t_{cleaned}"
227
- return cleaned
228
-
229
-
230
- def _dbt_names(tables: dict[str, TableDef]) -> dict[str, str]:
222
+ def _dbt_names(tables: Iterable[str]) -> dict[str, str]:
231
223
  """Each table key's dbt identifier, refusing two keys that normalise alike."""
232
- names: dict[str, str] = {}
233
- owners: dict[str, str] = {}
234
- for key in tables:
235
- name = _dbt_identifier(key)
236
- if name in owners:
237
- typer.echo(
238
- f"❌ Tables {owners[name]!r} and {key!r} would both be the dbt seed {name!r}. "
239
- "Rename one of them."
240
- )
241
- raise typer.Exit(1)
242
- owners[name] = key
243
- names[key] = name
244
- return names
224
+ try:
225
+ return dbt_names(tables)
226
+ except ValueError as exc:
227
+ typer.echo(f"❌ {exc}")
228
+ raise typer.Exit(1) from None
245
229
 
246
230
 
247
231
  @app.command(
@@ -420,6 +404,18 @@ def main(
420
404
  "--next",
421
405
  help="Shorthand for --days 1.",
422
406
  ),
407
+ defects_preset: Optional[str] = typer.Option(
408
+ None,
409
+ "--defects",
410
+ metavar="PRESET",
411
+ help=(
412
+ "Break the data on purpose: 'clean' (no preset, the default), 'messy' (a mix on\n"
413
+ "every table) or 'training' (each kind of dbt test fails once); a table's own\n"
414
+ "`defects` still apply. 'none' ignores every defect, the tables' own too.\n"
415
+ "Overrides the model's run.defects. Writes defects_report.json and\n"
416
+ "EXPECTED_FAILURES.md into the project."
417
+ ),
418
+ ),
423
419
  days_format: str = typer.Option(
424
420
  "batches",
425
421
  "--days-format",
@@ -463,6 +459,14 @@ def main(
463
459
  tolerance = _given(test_tolerance)
464
460
  tolerance = DEFAULT_TOLERANCE if tolerance is None else float(tolerance)
465
461
 
462
+ preset = _given(defects_preset)
463
+ if preset is not None:
464
+ preset = preset.lower()
465
+ if preset not in DEFECT_PRESETS:
466
+ raise typer.BadParameter(
467
+ f"Choose one of: {', '.join(DEFECT_PRESETS)}.", param_hint="--defects"
468
+ )
469
+
466
470
  # -------------------------
467
471
  # Read the model (names untouched)
468
472
  # -------------------------
@@ -537,7 +541,11 @@ def main(
537
541
  if table_seeds:
538
542
  rolled = ", ".join(f"{key}={value}" for key, value in sorted(table_seeds.items()))
539
543
  typer.echo(f"🎲 Re-rolling with a table seed of its own: {rolled}")
540
- dbt_names = _dbt_names(tables)
544
+ names = _dbt_names([*tables, *history_tables(inputs)])
545
+ plan = planned_defects(model, preset, days=days or 0)
546
+ preset = preset or (run.defects if run.defects is not None else None)
547
+ if preset in ("messy", "training") and not plan:
548
+ typer.echo(f"ℹ️ The {preset} preset finds nothing to break in this model: no defects.")
541
549
 
542
550
  project_name = normalize_identifier(_given(name) or model.name or _model_stem(file))
543
551
  dest = Path.cwd() / f"dbt_{project_name}"
@@ -596,22 +604,33 @@ def main(
596
604
  typer.echo(f"❌ {exc}")
597
605
  raise typer.Exit(1) from None
598
606
 
607
+ # -------------------------
608
+ # Defects: broken on purpose, after the clean data, and reported
609
+ # -------------------------
610
+ if plan:
611
+ typer.echo("🧨 Breaking the data on purpose (defects)...")
612
+ out = finish_run(
613
+ inputs,
614
+ day_results or generated_tables,
615
+ plan,
616
+ seed=seed,
617
+ preset=preset,
618
+ as_of=anchor,
619
+ hint_tests=hint_tests,
620
+ test_tolerance=tolerance,
621
+ names=names,
622
+ )
623
+ report = out.report
624
+ generated_tables = out.frames
625
+ if day_results:
626
+ day_results = out.days
627
+
599
628
  # -------------------------
600
629
  # dbt names: a table key becomes a [a-z0-9_] seed and model name here,
601
630
  # once, so the CSV's stem, `stg_<name>` and every `ref()` agree.
602
631
  # -------------------------
603
- dbt_tables = {
604
- dbt_names[key]: replace(table, name=dbt_names[key]) for key, table in tables.items()
605
- }
606
- dbt_refs = [
607
- {
608
- **ref,
609
- "source_table": dbt_names.get(ref["source_table"], ref["source_table"]),
610
- "target_table": dbt_names.get(ref["target_table"], ref["target_table"]),
611
- }
612
- for ref in refs
613
- ]
614
- dbt_frames = {dbt_names.get(key, key): df for key, df in generated_tables.items()}
632
+ dbt_tables, dbt_refs = for_dbt(out.tables, refs, names)
633
+ dbt_frames = {names.get(key, key): df for key, df in generated_tables.items()}
615
634
 
616
635
  # -------------------------
617
636
  # Write dbt seeds (normalized names)
@@ -623,7 +642,7 @@ def main(
623
642
 
624
643
  if day_results and days_format != "final":
625
644
  write = write_batches if days_format == "batches" else write_changelog
626
- write(dest, day_results, dbt_names)
645
+ write(dest, day_results, names)
627
646
 
628
647
  # -------------------------
629
648
  # Build dbt assets
@@ -651,6 +670,10 @@ def main(
651
670
  # Keep the original model file for reference
652
671
  shutil.copy(file, dest / file.name)
653
672
 
673
+ if report is not None:
674
+ write_defects_report(report, dest)
675
+ write_expected_failures(report, dest)
676
+
654
677
  # -------------------------
655
678
  # Summary
656
679
  # -------------------------
@@ -698,6 +721,9 @@ def main(
698
721
  for label in duplicate_unique:
699
722
  typer.echo(f" - {label}")
700
723
 
724
+ if report is not None:
725
+ _print_defects(report)
726
+
701
727
  # -------------------------
702
728
  # Done
703
729
  # -------------------------
@@ -707,6 +733,29 @@ def main(
707
733
  typer.echo(" dbt build # loads the seeds, builds the models, runs every test")
708
734
 
709
735
 
736
+ def _spec_of(file: Path) -> str:
737
+ """The spec a conforming model is written against: 0.3.0, or 0.2.0 (DBML included)."""
738
+ return "0.3.0" if str(load(file).version).startswith("0.3") else "0.2.0"
739
+
740
+
741
+ def _print_defects(report: DefectsReport) -> None:
742
+ typer.echo("\n🧨 Defects (defects_report.json, EXPECTED_FAILURES.md)")
743
+ for defect in report.defects:
744
+ column = defect.column if not isinstance(defect.column, list) else ", ".join(defect.column)
745
+ on = f".{column}" if column else ""
746
+ typer.echo(
747
+ f" {defect.table}{on}: {defect.defect}, {defect.applied} "
748
+ f"row{'' if defect.applied == 1 else 's'}"
749
+ )
750
+ if defect.note and not defect.applied:
751
+ typer.echo(f" ⚠️ {defect.note}")
752
+ count = len(report.expected_failures)
753
+ typer.echo(f" dbt build should fail {count} test{'' if count == 1 else 's'}:")
754
+ for failure in report.expected_failures:
755
+ warn = " (warn)" if failure.severity == "warn" else ""
756
+ typer.echo(f" - {failure.test}{warn}")
757
+
758
+
710
759
  @app.command("validate")
711
760
  def validate_command(
712
761
  file: Path = typer.Argument( # noqa: B008
@@ -718,7 +767,7 @@ def validate_command(
718
767
  help="The model: a .model2data.yml / .yaml / .json document, or a .dbml file.",
719
768
  ),
720
769
  ):
721
- """Check that a model conforms to spec 0.2.0, printing every issue with its path.
770
+ """Check that a model conforms to spec 0.3.0 (or 0.2.x), printing every issue with its path.
722
771
 
723
772
  Exits 1 when there is an error; warnings are printed, and the model conforms.
724
773
  """
@@ -726,7 +775,7 @@ def validate_command(
726
775
  _print_issues(file, issues)
727
776
  if any(issue.is_error for issue in issues):
728
777
  raise typer.Exit(1)
729
- typer.echo(f"✅ {file.name} conforms to spec 0.2.0.")
778
+ typer.echo(f"✅ {file.name} conforms to spec {_spec_of(file)}.")
730
779
 
731
780
 
732
781
  @app.command("convert")
@@ -748,7 +797,8 @@ def convert_command(
748
797
  ),
749
798
  force: bool = typer.Option(False, "--force", help="Overwrite the output file if it exists."),
750
799
  ):
751
- """Convert a model -- DBML, typically -- to a spec 0.2.0 .model2data.yml document."""
800
+ """Convert a model -- DBML, typically -- to a .model2data.yml document (spec 0.2.0, or
801
+ 0.3.0 when it has defects)."""
752
802
  try:
753
803
  model = load(file)
754
804
  except ModelError as error:
@@ -0,0 +1,61 @@
1
+ """A model's table keys as the seed and model names they get in a dbt project.
2
+
3
+ The CLI and `model2data.defects` both name tables this way, so a test name the
4
+ defects report predicts is the one dbt gives the test in the generated project.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import re
10
+ from collections.abc import Iterable, Mapping
11
+ from dataclasses import replace
12
+
13
+ from model2data.parse.dbml import TableDef
14
+
15
+
16
+ def dbt_identifier(key: str) -> str:
17
+ """A table key as the seed and model name it gets in the dbt project.
18
+
19
+ Spec 0.2.0 leaves a name as written (`user accounts`, `raw.orders`) and has
20
+ a consumer normalise it where it reaches a file or a dbt identifier, to
21
+ `[a-z0-9_]`. Underscores are kept as they are, so `_dlt_loads` and
22
+ `stories__kids` stay what they were.
23
+ """
24
+ cleaned = re.sub(r"[^0-9a-z_]+", "_", key.lower())
25
+ if cleaned.strip("_") == "":
26
+ cleaned = "table"
27
+ elif cleaned[0].isdigit():
28
+ cleaned = f"t_{cleaned}"
29
+ return cleaned
30
+
31
+
32
+ def dbt_names(keys: Iterable[str]) -> dict[str, str]:
33
+ """Each table key's dbt identifier; ValueError when two keys normalise alike."""
34
+ names: dict[str, str] = {}
35
+ owners: dict[str, str] = {}
36
+ for key in keys:
37
+ name = dbt_identifier(key)
38
+ if name in owners:
39
+ raise ValueError(
40
+ f"Tables {owners[name]!r} and {key!r} would both be the dbt seed {name!r}. "
41
+ "Rename one of them."
42
+ )
43
+ owners[name] = key
44
+ names[key] = name
45
+ return names
46
+
47
+
48
+ def for_dbt(
49
+ tables: Mapping[str, TableDef], refs: list[dict], names: Mapping[str, str]
50
+ ) -> tuple[dict[str, TableDef], list[dict]]:
51
+ """`(tables, refs)` renamed to their dbt names, as `generate_dbt_yml` takes them."""
52
+ dbt_tables = {names[key]: replace(table, name=names[key]) for key, table in tables.items()}
53
+ dbt_refs = [
54
+ {
55
+ **ref,
56
+ "source_table": names.get(ref["source_table"], ref["source_table"]),
57
+ "target_table": names.get(ref["target_table"], ref["target_table"]),
58
+ }
59
+ for ref in refs
60
+ ]
61
+ return dbt_tables, dbt_refs
@@ -0,0 +1,38 @@
1
+ {#-
2
+ Generic tests of a history table (`incremental.history`): one row per version
3
+ of each key, valid from `valid_from` until `valid_to` (null for the current
4
+ version). Written only into a project that has a history table. Plain SQL, no
5
+ dbt package. Each test returns the keys (or versions) that break it.
6
+ -#}
7
+
8
+ {% test model2data_one_current_row(model, key, current) %}
9
+ {%- set keys = [] -%}
10
+ {%- for column in key -%}{%- do keys.append(adapter.quote(column)) -%}{%- endfor %}
11
+ select {{ keys | join(', ') }},
12
+ sum(case when {{ adapter.quote(current) }} then 1 else 0 end) as current_rows
13
+ from {{ model }}
14
+ group by {{ keys | join(', ') }}
15
+ having sum(case when {{ adapter.quote(current) }} then 1 else 0 end) <> 1
16
+ {% endtest %}
17
+
18
+
19
+ {#-
20
+ A version overlaps the next one of its key when it is still valid when the
21
+ next begins: no `valid_to`, or one after the next version's `valid_from`.
22
+ -#}
23
+ {% test model2data_no_overlapping_ranges(model, key, valid_from, valid_to) %}
24
+ {%- set keys = [] -%}
25
+ {%- for column in key -%}{%- do keys.append(adapter.quote(column)) -%}{%- endfor %}
26
+ select *
27
+ from (
28
+ select {{ keys | join(', ') }},
29
+ {{ adapter.quote(valid_from) }} as version_from,
30
+ {{ adapter.quote(valid_to) }} as version_to,
31
+ lead({{ adapter.quote(valid_from) }}) over (
32
+ partition by {{ keys | join(', ') }} order by {{ adapter.quote(valid_from) }}
33
+ ) as next_from
34
+ from {{ model }}
35
+ ) as versions
36
+ where next_from is not null
37
+ and (version_to is null or version_to > next_from)
38
+ {% endtest %}