model2data 1.9.0__tar.gz → 1.10.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. {model2data-1.9.0/model2data.egg-info → model2data-1.10.1}/PKG-INFO +7 -1
  2. {model2data-1.9.0 → model2data-1.10.1}/README.md +48 -0
  3. {model2data-1.9.0 → model2data-1.10.1}/README_PYPI.md +6 -0
  4. {model2data-1.9.0 → model2data-1.10.1}/model2data/cli.py +87 -11
  5. {model2data-1.9.0 → model2data-1.10.1}/model2data/model/_yaml.py +66 -0
  6. {model2data-1.9.0 → model2data-1.10.1}/model2data/model/reader.py +11 -4
  7. {model2data-1.9.0 → model2data-1.10.1/model2data.egg-info}/PKG-INFO +7 -1
  8. {model2data-1.9.0 → model2data-1.10.1}/model2data.egg-info/SOURCES.txt +1 -0
  9. {model2data-1.9.0 → model2data-1.10.1}/pyproject.toml +1 -1
  10. {model2data-1.9.0 → model2data-1.10.1}/tests/test_cli_model.py +134 -2
  11. model2data-1.10.1/tests/test_issue_lines.py +87 -0
  12. {model2data-1.9.0 → model2data-1.10.1}/tests/test_model.py +3 -1
  13. {model2data-1.9.0 → model2data-1.10.1}/LICENSE +0 -0
  14. {model2data-1.9.0 → model2data-1.10.1}/model2data/__init__.py +0 -0
  15. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/__init__.py +0 -0
  16. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/LICENSE +0 -0
  17. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/VENDORED.md +0 -0
  18. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/__init__.py +0 -0
  19. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/_classes/__init__.py +0 -0
  20. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/_classes/base.py +0 -0
  21. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/_classes/column.py +0 -0
  22. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/_classes/enum.py +0 -0
  23. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/_classes/expression.py +0 -0
  24. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/_classes/index.py +0 -0
  25. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/_classes/note.py +0 -0
  26. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/_classes/project.py +0 -0
  27. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/_classes/reference.py +0 -0
  28. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/_classes/sticky_note.py +0 -0
  29. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/_classes/table.py +0 -0
  30. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/_classes/table_group.py +0 -0
  31. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/classes/__init__.py +0 -0
  32. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/constants.py +0 -0
  33. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/database.py +0 -0
  34. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/definitions/__init__.py +0 -0
  35. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/definitions/column.py +0 -0
  36. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/definitions/common.py +0 -0
  37. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/definitions/enum.py +0 -0
  38. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/definitions/generic.py +0 -0
  39. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/definitions/index.py +0 -0
  40. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/definitions/project.py +0 -0
  41. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/definitions/records.py +0 -0
  42. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/definitions/reference.py +0 -0
  43. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/definitions/sticky_note.py +0 -0
  44. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/definitions/table.py +0 -0
  45. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/definitions/table_group.py +0 -0
  46. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/definitions/table_partial.py +0 -0
  47. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/exceptions.py +0 -0
  48. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/parser/__init__.py +0 -0
  49. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/parser/blueprints.py +0 -0
  50. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/parser/parser.py +0 -0
  51. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/__init__.py +0 -0
  52. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/base.py +0 -0
  53. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/dbml/__init__.py +0 -0
  54. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/dbml/default/__init__.py +0 -0
  55. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/dbml/default/column.py +0 -0
  56. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/dbml/default/enum.py +0 -0
  57. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/dbml/default/expression.py +0 -0
  58. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/dbml/default/index.py +0 -0
  59. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/dbml/default/note.py +0 -0
  60. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/dbml/default/project.py +0 -0
  61. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/dbml/default/reference.py +0 -0
  62. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/dbml/default/renderer.py +0 -0
  63. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/dbml/default/sticky_note.py +0 -0
  64. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/dbml/default/table.py +0 -0
  65. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/dbml/default/table_group.py +0 -0
  66. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/dbml/default/utils.py +0 -0
  67. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/sql/__init__.py +0 -0
  68. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/sql/default/__init__.py +0 -0
  69. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/sql/default/column.py +0 -0
  70. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/sql/default/enum.py +0 -0
  71. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/sql/default/expression.py +0 -0
  72. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/sql/default/index.py +0 -0
  73. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/sql/default/note.py +0 -0
  74. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/sql/default/reference.py +0 -0
  75. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/sql/default/renderer.py +0 -0
  76. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/sql/default/table.py +0 -0
  77. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/renderer/sql/default/utils.py +0 -0
  78. {model2data-1.9.0 → model2data-1.10.1}/model2data/_vendor/pydbml/tools.py +0 -0
  79. {model2data-1.9.0 → model2data-1.10.1}/model2data/dbt/__init__.py +0 -0
  80. {model2data-1.9.0 → model2data-1.10.1}/model2data/dbt/hint_tests.py +0 -0
  81. {model2data-1.9.0 → model2data-1.10.1}/model2data/dbt/naming.py +0 -0
  82. {model2data-1.9.0 → model2data-1.10.1}/model2data/dbt/project.py +0 -0
  83. {model2data-1.9.0 → model2data-1.10.1}/model2data/dbt/templates/dbt_project.yml.jinja +0 -0
  84. {model2data-1.9.0 → model2data-1.10.1}/model2data/dbt/templates/hint_macros/model2data_hint_tests.sql +0 -0
  85. {model2data-1.9.0 → model2data-1.10.1}/model2data/dbt/templates/history_macros/model2data_history_tests.sql +0 -0
  86. {model2data-1.9.0 → model2data-1.10.1}/model2data/dbt/templates/macros/generate_schema_name.sql +0 -0
  87. {model2data-1.9.0 → model2data-1.10.1}/model2data/dbt/templates/profiles.yml.jinja +0 -0
  88. {model2data-1.9.0 → model2data-1.10.1}/model2data/dbt/tests.py +0 -0
  89. {model2data-1.9.0 → model2data-1.10.1}/model2data/defects/__init__.py +0 -0
  90. {model2data-1.9.0 → model2data-1.10.1}/model2data/defects/apply.py +0 -0
  91. {model2data-1.9.0 → model2data-1.10.1}/model2data/defects/checks.py +0 -0
  92. {model2data-1.9.0 → model2data-1.10.1}/model2data/defects/presets.py +0 -0
  93. {model2data-1.9.0 → model2data-1.10.1}/model2data/defects/report.py +0 -0
  94. {model2data-1.9.0 → model2data-1.10.1}/model2data/generate/__init__.py +0 -0
  95. {model2data-1.9.0 → model2data-1.10.1}/model2data/generate/core.py +0 -0
  96. {model2data-1.9.0 → model2data-1.10.1}/model2data/generate/days.py +0 -0
  97. {model2data-1.9.0 → model2data-1.10.1}/model2data/generate/faker.py +0 -0
  98. {model2data-1.9.0 → model2data-1.10.1}/model2data/generate/hints.py +0 -0
  99. {model2data-1.9.0 → model2data-1.10.1}/model2data/generate/history.py +0 -0
  100. {model2data-1.9.0 → model2data-1.10.1}/model2data/generate/kinds.py +0 -0
  101. {model2data-1.9.0 → model2data-1.10.1}/model2data/generate/options.py +0 -0
  102. {model2data-1.9.0 → model2data-1.10.1}/model2data/generate/relationships.py +0 -0
  103. {model2data-1.9.0 → model2data-1.10.1}/model2data/generate/timeline.py +0 -0
  104. {model2data-1.9.0 → model2data-1.10.1}/model2data/model/__init__.py +0 -0
  105. {model2data-1.9.0 → model2data-1.10.1}/model2data/model/dbml.py +0 -0
  106. {model2data-1.9.0 → model2data-1.10.1}/model2data/model/document.py +0 -0
  107. {model2data-1.9.0 → model2data-1.10.1}/model2data/model/dump.py +0 -0
  108. {model2data-1.9.0 → model2data-1.10.1}/model2data/model/engine.py +0 -0
  109. {model2data-1.9.0 → model2data-1.10.1}/model2data/model/errors.py +0 -0
  110. {model2data-1.9.0 → model2data-1.10.1}/model2data/model/types.py +0 -0
  111. {model2data-1.9.0 → model2data-1.10.1}/model2data/model/validate.py +0 -0
  112. {model2data-1.9.0 → model2data-1.10.1}/model2data/output.py +0 -0
  113. {model2data-1.9.0 → model2data-1.10.1}/model2data/parse/__init__.py +0 -0
  114. {model2data-1.9.0 → model2data-1.10.1}/model2data/parse/dbml.py +0 -0
  115. {model2data-1.9.0 → model2data-1.10.1}/model2data/spec/PUBLISHING.md +0 -0
  116. {model2data-1.9.0 → model2data-1.10.1}/model2data/spec/README.md +0 -0
  117. {model2data-1.9.0 → model2data-1.10.1}/model2data/spec/examples/coffee_webshop.model2data.yml +0 -0
  118. {model2data-1.9.0 → model2data-1.10.1}/model2data/spec/model.schema.json +0 -0
  119. {model2data-1.9.0 → model2data-1.10.1}/model2data/utils.py +0 -0
  120. {model2data-1.9.0 → model2data-1.10.1}/model2data.egg-info/dependency_links.txt +0 -0
  121. {model2data-1.9.0 → model2data-1.10.1}/model2data.egg-info/entry_points.txt +0 -0
  122. {model2data-1.9.0 → model2data-1.10.1}/model2data.egg-info/requires.txt +0 -0
  123. {model2data-1.9.0 → model2data-1.10.1}/model2data.egg-info/top_level.txt +0 -0
  124. {model2data-1.9.0 → model2data-1.10.1}/setup.cfg +0 -0
  125. {model2data-1.9.0 → model2data-1.10.1}/tests/test_as_of_anchor.py +0 -0
  126. {model2data-1.9.0 → model2data-1.10.1}/tests/test_cli.py +0 -0
  127. {model2data-1.9.0 → model2data-1.10.1}/tests/test_column_time_hints.py +0 -0
  128. {model2data-1.9.0 → model2data-1.10.1}/tests/test_coverage_gaps.py +0 -0
  129. {model2data-1.9.0 → model2data-1.10.1}/tests/test_days.py +0 -0
  130. {model2data-1.9.0 → model2data-1.10.1}/tests/test_dbml_parser.py +0 -0
  131. {model2data-1.9.0 → model2data-1.10.1}/tests/test_dbml_parser_fuzz.py +0 -0
  132. {model2data-1.9.0 → model2data-1.10.1}/tests/test_dbt_integration.py +0 -0
  133. {model2data-1.9.0 → model2data-1.10.1}/tests/test_dbt_naming.py +0 -0
  134. {model2data-1.9.0 → model2data-1.10.1}/tests/test_dbt_project.py +0 -0
  135. {model2data-1.9.0 → model2data-1.10.1}/tests/test_dbt_tests.py +0 -0
  136. {model2data-1.9.0 → model2data-1.10.1}/tests/test_defects.py +0 -0
  137. {model2data-1.9.0 → model2data-1.10.1}/tests/test_defects_dbt.py +0 -0
  138. {model2data-1.9.0 → model2data-1.10.1}/tests/test_defects_replay.py +0 -0
  139. {model2data-1.9.0 → model2data-1.10.1}/tests/test_defects_report.py +0 -0
  140. {model2data-1.9.0 → model2data-1.10.1}/tests/test_defects_review.py +0 -0
  141. {model2data-1.9.0 → model2data-1.10.1}/tests/test_distributions.py +0 -0
  142. {model2data-1.9.0 → model2data-1.10.1}/tests/test_examples_model.py +0 -0
  143. {model2data-1.9.0 → model2data-1.10.1}/tests/test_faker_name_inference.py +0 -0
  144. {model2data-1.9.0 → model2data-1.10.1}/tests/test_generation.py +0 -0
  145. {model2data-1.9.0 → model2data-1.10.1}/tests/test_hint_tests.py +0 -0
  146. {model2data-1.9.0 → model2data-1.10.1}/tests/test_history.py +0 -0
  147. {model2data-1.9.0 → model2data-1.10.1}/tests/test_lone_country.py +0 -0
  148. {model2data-1.9.0 → model2data-1.10.1}/tests/test_model_dbml.py +0 -0
  149. {model2data-1.9.0 → model2data-1.10.1}/tests/test_model_edges.py +0 -0
  150. {model2data-1.9.0 → model2data-1.10.1}/tests/test_model_messages.py +0 -0
  151. {model2data-1.9.0 → model2data-1.10.1}/tests/test_model_vocabulary.py +0 -0
  152. {model2data-1.9.0 → model2data-1.10.1}/tests/test_options.py +0 -0
  153. {model2data-1.9.0 → model2data-1.10.1}/tests/test_release_stress.py +0 -0
  154. {model2data-1.9.0 → model2data-1.10.1}/tests/test_reproducibility.py +0 -0
  155. {model2data-1.9.0 → model2data-1.10.1}/tests/test_row_identity.py +0 -0
  156. {model2data-1.9.0 → model2data-1.10.1}/tests/test_shaping.py +0 -0
  157. {model2data-1.9.0 → model2data-1.10.1}/tests/test_spec_alignment.py +0 -0
  158. {model2data-1.9.0 → model2data-1.10.1}/tests/test_spec_schema.py +0 -0
  159. {model2data-1.9.0 → model2data-1.10.1}/tests/test_table_seeds.py +0 -0
  160. {model2data-1.9.0 → model2data-1.10.1}/tests/test_timeline.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: model2data
3
- Version: 1.9.0
3
+ Version: 1.10.1
4
4
  Summary: Generate analytics-ready datasets from DBML models
5
5
  Author: JB Analytica
6
6
  License-Expression: MIT
@@ -177,6 +177,12 @@ run:
177
177
  CLI option overrides the one it names. `model2data validate FILE` checks a model and prints every
178
178
  issue with its path in the document.
179
179
 
180
+ **Validate models in CI.** `model2data validate a.model2data.yml b.model2data.yml` (or
181
+ `--glob "**/*.model2data.yml"`, `--format github` for inline pull request annotations) exits 1
182
+ when any model has an error. A GitHub Action (`uses: JB-Analytica/model2data@v1`), a pre-commit
183
+ hook (`model2data-validate`) and a GitLab CI snippet are in the
184
+ [README](https://github.com/JB-Analytica/model2data#validate-models-in-ci).
185
+
180
186
  **DBML is supported input.** `--file` also takes a `.dbml` file, converted to the same model
181
187
  before generating; a JSON note on a column (`[note: '{"min": 1}']`) becomes its `generate`, any
182
188
  other note its description. `model2data convert schema.dbml -o schema.model2data.yml` writes the
@@ -199,6 +199,54 @@ unquoted non-ASCII names — can't be read yet, and are refused with the line an
199
199
 
200
200
  ---
201
201
 
202
+ ## Validate models in CI
203
+
204
+ Teams that keep `*.model2data.yml` models in git can fail a pull request that breaks one.
205
+ `model2data validate` takes several files and `--glob` patterns, prefixes each issue with its
206
+ file, exits 1 if any model has an error, and with `--format github` prints annotations that show
207
+ inline on the pull request's Files tab. `--require-files` fails when nothing matched.
208
+
209
+ **GitHub Actions**
210
+
211
+ ```yaml
212
+ name: model2data
213
+ on:
214
+ pull_request:
215
+ paths: ["**/*.model2data.yml"]
216
+ jobs:
217
+ validate:
218
+ runs-on: ubuntu-latest
219
+ steps:
220
+ - uses: actions/checkout@v5
221
+ - uses: JB-Analytica/model2data@v1
222
+ ```
223
+
224
+ Inputs: `files` (glob, default `**/*.model2data.yml`), `version` (default: the release the tag
225
+ points at, e.g. `1.10.1`; a specifier such as `>=1.10,<2` also works), `python-version` (3.12).
226
+
227
+ **pre-commit**
228
+
229
+ ```yaml
230
+ repos:
231
+ - repo: https://github.com/JB-Analytica/model2data
232
+ rev: v1.10.1
233
+ hooks:
234
+ - id: model2data-validate
235
+ ```
236
+
237
+ **GitLab CI**
238
+
239
+ ```yaml
240
+ model2data:
241
+ image: python:3.12-slim
242
+ rules:
243
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
244
+ changes: ["**/*.model2data.yml"]
245
+ script:
246
+ - pip install model2data==1.10.1
247
+ - model2data validate --glob "**/*.model2data.yml"
248
+ ```
249
+
202
250
  ## model2data studio — the same engine, in the browser
203
251
 
204
252
  [**model2data studio**](https://studio.jbanalytica.com) puts everything on this page behind a
@@ -130,6 +130,12 @@ run:
130
130
  CLI option overrides the one it names. `model2data validate FILE` checks a model and prints every
131
131
  issue with its path in the document.
132
132
 
133
+ **Validate models in CI.** `model2data validate a.model2data.yml b.model2data.yml` (or
134
+ `--glob "**/*.model2data.yml"`, `--format github` for inline pull request annotations) exits 1
135
+ when any model has an error. A GitHub Action (`uses: JB-Analytica/model2data@v1`), a pre-commit
136
+ hook (`model2data-validate`) and a GitLab CI snippet are in the
137
+ [README](https://github.com/JB-Analytica/model2data#validate-models-in-ci).
138
+
133
139
  **DBML is supported input.** `--file` also takes a `.dbml` file, converted to the same model
134
140
  before generating; a JSON note on a column (`[note: '{"min": 1}']`) becomes its `generate`, any
135
141
  other note its description. `model2data convert schema.dbml -o schema.model2data.yml` writes the
@@ -196,15 +196,16 @@ def _read_model(file: Path) -> Model:
196
196
  return model
197
197
 
198
198
 
199
- def _print_issues(file: Path, issues: list[Issue]) -> None:
199
+ def _print_issues(file: Path, issues: list[Issue], label: Optional[str] = None) -> None:
200
200
  """Errors, then warnings, each with its document path."""
201
+ label = label or file.name
201
202
  errors = [issue for issue in issues if issue.is_error]
202
203
  warnings = [issue for issue in issues if not issue.is_error]
203
204
  for marker, group, word in (("❌", errors, "error"), ("⚠️ ", warnings, "warning")):
204
205
  if not group:
205
206
  continue
206
207
  count = f"1 {word}" if len(group) == 1 else f"{len(group)} {word}s"
207
- typer.echo(f"{marker} {file.name}: {count}")
208
+ typer.echo(f"{marker} {label}: {count}")
208
209
  for issue in group:
209
210
  text = str(issue).removeprefix("warning: ").replace("\n", "\n ")
210
211
  typer.echo(f" - {text}")
@@ -756,26 +757,101 @@ def _print_defects(report: DefectsReport) -> None:
756
757
  typer.echo(f" - {failure.test}{warn}")
757
758
 
758
759
 
760
+ _SKIPPED_DIRS = {".git", ".venv", "node_modules"}
761
+
762
+
763
+ def _expand_globs(patterns: list[str]) -> list[Path]:
764
+ """Files matching recursive glob patterns relative to the cwd, sorted, without duplicates."""
765
+ found: dict[Path, None] = {}
766
+ for pattern in patterns:
767
+ for match in sorted(Path.cwd().glob(pattern)):
768
+ relative = match.relative_to(Path.cwd())
769
+ if match.is_file() and not _SKIPPED_DIRS.intersection(relative.parts):
770
+ found[relative] = None
771
+ return list(found)
772
+
773
+
774
+ def _github_escape(text: str, *, prop: bool = False) -> str:
775
+ """Escape a workflow-command message (or property value) the way GitHub expects."""
776
+ text = text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
777
+ return text.replace(":", "%3A").replace(",", "%2C") if prop else text
778
+
779
+
780
+ def _print_github_issues(file: Path, issues: list[Issue]) -> None:
781
+ """One `::error` / `::warning` annotation per issue, shown inline on a pull request."""
782
+ for issue in issues:
783
+ level = "error" if issue.is_error else "warning"
784
+ where = f"file={_github_escape(file.as_posix(), prop=True)}"
785
+ if issue.line:
786
+ where += f",line={issue.line}"
787
+ text = f"{issue.path}: {issue.message}" if issue.path else issue.message
788
+ typer.echo(f"::{level} {where}::{_github_escape(text)}")
789
+
790
+
759
791
  @app.command("validate")
760
792
  def validate_command(
761
- file: Path = typer.Argument( # noqa: B008
762
- ...,
793
+ files: Optional[list[Path]] = typer.Argument( # noqa: B008
794
+ None,
763
795
  exists=True,
764
796
  file_okay=True,
765
797
  dir_okay=False,
766
798
  readable=True,
767
- help="The model: a .model2data.yml / .yaml / .json document, or a .dbml file.",
799
+ help="The models: .model2data.yml / .yaml / .json documents, or .dbml files.",
800
+ ),
801
+ glob: Optional[list[str]] = typer.Option( # noqa: B008
802
+ None,
803
+ "--glob",
804
+ help=(
805
+ "A recursive glob relative to the current directory, e.g. '**/*.model2data.yml'. "
806
+ "Repeatable; expands without the shell, so CI needs no globstar."
807
+ ),
808
+ ),
809
+ output_format: str = typer.Option(
810
+ "text",
811
+ "--format",
812
+ help="'text' (default) or 'github' for ::error / ::warning annotations on a pull request.",
813
+ ),
814
+ require_files: bool = typer.Option(
815
+ False, "--require-files", help="Exit 1 when no model file was given or matched."
768
816
  ),
769
817
  ):
770
- """Check that a model conforms to spec 0.3.0 (or 0.2.x), printing every issue with its path.
818
+ """Check that models conform to spec 0.3.0 (or 0.2.x), printing every issue with its path.
771
819
 
772
- Exits 1 when there is an error; warnings are printed, and the model conforms.
820
+ Takes one or more files and/or --glob patterns. Exits 1 when any file has an error;
821
+ warnings are printed, and the model conforms.
773
822
  """
774
- issues = validate(file)
775
- _print_issues(file, issues)
776
- if any(issue.is_error for issue in issues):
823
+ if output_format not in ("text", "github"):
824
+ raise typer.BadParameter("must be 'text' or 'github'", param_hint="--format")
825
+ targets = list(files or [])
826
+ if glob:
827
+ targets += [path for path in _expand_globs(glob) if path not in targets]
828
+ if not targets:
829
+ typer.echo("no model files matched")
830
+ raise typer.Exit(1 if require_files else 0)
831
+
832
+ failed = 0
833
+ for file in targets:
834
+ issues = validate(file)
835
+ if output_format == "github":
836
+ _print_github_issues(file, issues)
837
+ else:
838
+ _print_issues(file, issues, label=str(file) if len(targets) > 1 else file.name)
839
+ if any(issue.is_error for issue in issues):
840
+ failed += 1
841
+ elif output_format == "text":
842
+ typer.echo(
843
+ f"✅ {file if len(targets) > 1 else file.name} conforms to spec {_spec_of(file)}."
844
+ )
845
+ if len(targets) > 1 or output_format == "github":
846
+ count = len(targets)
847
+ checked = f"{count} model file{'s' if count != 1 else ''}"
848
+ if failed:
849
+ verb = "does" if failed == 1 else "do"
850
+ typer.echo(f"❌ {failed} of {checked} {verb} not conform.")
851
+ else:
852
+ typer.echo(f"✅ {checked} {'conforms' if count == 1 else 'conform'}.")
853
+ if failed:
777
854
  raise typer.Exit(1)
778
- typer.echo(f"✅ {file.name} conforms to spec {_spec_of(file)}.")
779
855
 
780
856
 
781
857
  @app.command("convert")
@@ -23,6 +23,7 @@ from __future__ import annotations
23
23
 
24
24
  import math
25
25
  import re
26
+ from dataclasses import replace
26
27
  from typing import Any
27
28
 
28
29
  import yaml
@@ -160,6 +161,71 @@ def _check_mappings(node: Node, path: list[PathPart], issues: list[Issue]) -> No
160
161
  _check_mappings(item, [*path, index], issues)
161
162
 
162
163
 
164
+ def locate_issues(text: str, issues: list[Issue]) -> list[Issue]:
165
+ """`issues` with a `line` on each one that has none, from where its path sits in `text`.
166
+
167
+ `text` is a YAML (or JSON) document; it is composed once, with the profile's
168
+ own loader so the node tree is the one `parse_yaml` read, and each issue's
169
+ dotted path is walked down it. A path that stops resolving (a key that is
170
+ missing, say) takes the line of the deepest node that did. Text that does
171
+ not compose leaves the issues as they are.
172
+ """
173
+ if not any(issue.line is None and issue.path for issue in issues):
174
+ return issues
175
+ loader = _CoreLoader(text)
176
+ try:
177
+ root = loader.get_single_node()
178
+ except Exception:
179
+ return issues
180
+ finally:
181
+ loader.dispose()
182
+ if root is None:
183
+ return issues
184
+ located = []
185
+ for issue in issues:
186
+ if issue.line is None and issue.path:
187
+ line = _line_of_path(root, issue.path)
188
+ if line is not None:
189
+ issue = replace(issue, line=line)
190
+ located.append(issue)
191
+ return located
192
+
193
+
194
+ def _line_of_path(root: Node, path: str) -> int | None:
195
+ """The line of the deepest node `path` reaches, or None if it reaches only the root."""
196
+ node = root
197
+ rest = path
198
+ line: int | None = None
199
+ while rest:
200
+ if isinstance(node, MappingNode):
201
+ best: tuple[Node, Node] | None = None
202
+ best_len = 0
203
+ for key_node, value_node in node.value:
204
+ key = _key_text(key_node)
205
+ if not isinstance(key, str) or not key or len(key) <= best_len:
206
+ continue
207
+ if rest == key or rest.startswith(key + "."):
208
+ best, best_len = (key_node, value_node), len(key)
209
+ if best is None:
210
+ break
211
+ line = best[0].start_mark.line + 1
212
+ node = best[1]
213
+ rest = rest[best_len + 1 :]
214
+ elif isinstance(node, SequenceNode):
215
+ match = re.match(r"^(?:\[(\d+)\]|(\d+))(?:\.|$)", rest)
216
+ if match is None:
217
+ break
218
+ index = int(match.group(1) or match.group(2))
219
+ if index >= len(node.value):
220
+ break
221
+ node = node.value[index]
222
+ line = node.start_mark.line + 1
223
+ rest = rest[match.end() :]
224
+ else:
225
+ break
226
+ return line
227
+
228
+
163
229
  def parse_yaml(text: str) -> Any:
164
230
  """Read one YAML document under the spec's profile, or raise `ModelError`."""
165
231
  loader = _CoreLoader(text)
@@ -7,7 +7,7 @@ import os
7
7
  from pathlib import Path
8
8
  from typing import Literal, Optional, Union
9
9
 
10
- from model2data.model._yaml import parse_yaml
10
+ from model2data.model._yaml import locate_issues, parse_yaml
11
11
  from model2data.model.document import from_dict
12
12
  from model2data.model.errors import Issue, ModelError
13
13
  from model2data.model.types import Model
@@ -78,9 +78,16 @@ def load(source: Source, *, format: Optional[Format] = None) -> Model:
78
78
  from model2data.model.dbml import from_dbml
79
79
 
80
80
  return from_dbml(text)
81
- if format == "json":
82
- return from_dict(_parse_json(text))
83
- return from_dict(parse_yaml(text))
81
+ document = _parse_json(text) if format == "json" else parse_yaml(text)
82
+ try:
83
+ model = from_dict(document)
84
+ except ModelError as error:
85
+ # One composition of the text, however many issues: each gets its line.
86
+ raise ModelError(
87
+ locate_issues(text, error.issues), locate_issues(text, error.warnings)
88
+ ) from None
89
+ model.warnings = locate_issues(text, model.warnings)
90
+ return model
84
91
 
85
92
 
86
93
  def _parse_json(text: str):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: model2data
3
- Version: 1.9.0
3
+ Version: 1.10.1
4
4
  Summary: Generate analytics-ready datasets from DBML models
5
5
  Author: JB Analytica
6
6
  License-Expression: MIT
@@ -177,6 +177,12 @@ run:
177
177
  CLI option overrides the one it names. `model2data validate FILE` checks a model and prints every
178
178
  issue with its path in the document.
179
179
 
180
+ **Validate models in CI.** `model2data validate a.model2data.yml b.model2data.yml` (or
181
+ `--glob "**/*.model2data.yml"`, `--format github` for inline pull request annotations) exits 1
182
+ when any model has an error. A GitHub Action (`uses: JB-Analytica/model2data@v1`), a pre-commit
183
+ hook (`model2data-validate`) and a GitLab CI snippet are in the
184
+ [README](https://github.com/JB-Analytica/model2data#validate-models-in-ci).
185
+
180
186
  **DBML is supported input.** `--file` also takes a `.dbml` file, converted to the same model
181
187
  before generating; a JSON note on a column (`[note: '{"min": 1}']`) becomes its `generate`, any
182
188
  other note its description. `model2data convert schema.dbml -o schema.model2data.yml` writes the
@@ -140,6 +140,7 @@ tests/test_faker_name_inference.py
140
140
  tests/test_generation.py
141
141
  tests/test_hint_tests.py
142
142
  tests/test_history.py
143
+ tests/test_issue_lines.py
143
144
  tests/test_lone_country.py
144
145
  tests/test_model.py
145
146
  tests/test_model_dbml.py
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "model2data"
7
- version = "1.9.0"
7
+ version = "1.10.1"
8
8
  description = "Generate analytics-ready datasets from DBML models"
9
9
  readme = "README_PYPI.md"
10
10
  requires-python = ">=3.10"
@@ -187,8 +187,8 @@ def test_validate_prints_every_issue_with_its_path_and_exits_1(tmp_path):
187
187
  result = runner.invoke(app, ["validate", str(model)])
188
188
  assert result.exit_code == 1
189
189
  assert "❌ m.model2data.yml: 2 errors" in result.output
190
- assert " - tables.orders.columns.customer_id.nul: unknown key" in result.output
191
- assert " - run.rows: must be 1 or more (got 0)" in result.output
190
+ assert " - tables.orders.columns.customer_id.nul (line 16): unknown key" in result.output
191
+ assert " - run.rows (line 21): must be 1 or more (got 0)" in result.output
192
192
 
193
193
 
194
194
  def test_validate_exits_0_on_warnings_alone(tmp_path):
@@ -246,3 +246,135 @@ def test_help_lists_the_commands():
246
246
  assert result.exit_code == 0
247
247
  for command in ("generate", "validate", "convert"):
248
248
  assert command in result.output
249
+
250
+
251
+ # ---------------------------------------------------------------------------
252
+ # validate: several files, --glob, --format github, and the CI integration files
253
+ # ---------------------------------------------------------------------------
254
+ def _broken(tmp_path, name="bad.model2data.yml"):
255
+ return _write(tmp_path, name, MODEL.replace("rows: 12", "rows: 0"))
256
+
257
+
258
+ def test_validate_several_files_all_valid(tmp_path):
259
+ a = _write(tmp_path, "a.model2data.yml", MODEL)
260
+ b = _write(tmp_path, "b.model2data.yml", MODEL)
261
+ result = runner.invoke(app, ["validate", str(a), str(b)])
262
+ assert result.exit_code == 0, result.output
263
+ assert f"✅ {a} conforms to spec 0.2.0." in result.output
264
+ assert "✅ 2 model files conform." in result.output
265
+
266
+
267
+ def test_validate_several_files_one_invalid_exits_1_with_path_prefixed_issues(tmp_path):
268
+ good = _write(tmp_path, "good.model2data.yml", MODEL)
269
+ bad = _broken(tmp_path)
270
+ result = runner.invoke(app, ["validate", str(good), str(bad)])
271
+ assert result.exit_code == 1
272
+ assert f"❌ {bad}: 1 error" in result.output
273
+ assert " - run.rows (line 21): must be 1 or more (got 0)" in result.output
274
+ assert "❌ 1 of 2 model files does not conform." in result.output
275
+
276
+
277
+ def test_validate_glob_matches_nested_files(tmp_path, monkeypatch):
278
+ (tmp_path / "deep" / "er").mkdir(parents=True)
279
+ _write(tmp_path, "top.model2data.yml", MODEL)
280
+ _write(tmp_path / "deep" / "er", "nested.model2data.yml", MODEL)
281
+ (tmp_path / "node_modules").mkdir()
282
+ _write(tmp_path / "node_modules", "skipped.model2data.yml", MODEL)
283
+ monkeypatch.chdir(tmp_path)
284
+ result = runner.invoke(app, ["validate", "--glob", "**/*.model2data.yml"])
285
+ assert result.exit_code == 0, result.output
286
+ assert "top.model2data.yml" in result.output
287
+ assert "nested.model2data.yml" in result.output
288
+ assert "skipped" not in result.output
289
+ assert "2 model files conform" in result.output
290
+
291
+
292
+ def test_validate_glob_with_no_match_is_a_note_unless_files_are_required(tmp_path, monkeypatch):
293
+ monkeypatch.chdir(tmp_path)
294
+ result = runner.invoke(app, ["validate", "--glob", "**/*.model2data.yml"])
295
+ assert result.exit_code == 0
296
+ assert "no model files matched" in result.output
297
+ required = runner.invoke(app, ["validate", "--glob", "**/*.model2data.yml", "--require-files"])
298
+ assert required.exit_code == 1
299
+ assert "no model files matched" in required.output
300
+
301
+
302
+ def test_validate_format_github_prints_annotations(tmp_path, monkeypatch):
303
+ _broken(tmp_path)
304
+ monkeypatch.chdir(tmp_path)
305
+ result = runner.invoke(app, ["validate", "bad.model2data.yml", "--format", "github"])
306
+ assert result.exit_code == 1
307
+ assert (
308
+ "::error file=bad.model2data.yml,line=21::run.rows: must be 1 or more (got 0)"
309
+ in result.output
310
+ )
311
+ yaml_error = _write(tmp_path, "dup.model2data.yml", MODEL + "name: again\n")
312
+ result = runner.invoke(app, ["validate", yaml_error.name, "--format", "github"])
313
+ assert "::error file=dup.model2data.yml,line=25::" in result.output
314
+
315
+
316
+ def test_validate_format_github_prints_warnings_and_exits_0():
317
+ result = runner.invoke(
318
+ app, ["validate", str(EXAMPLES / "hackernews.dbml"), "--format", "github"]
319
+ )
320
+ assert result.exit_code == 0
321
+ assert "::warning file=" in result.output
322
+
323
+
324
+ def test_validate_rejects_an_unknown_format(tmp_path):
325
+ model = _write(tmp_path, "m.model2data.yml", MODEL)
326
+ assert runner.invoke(app, ["validate", str(model), "--format", "xml"]).exit_code != 0
327
+
328
+
329
+ def test_ci_integration_files_run_the_validate_command():
330
+ import yaml
331
+
332
+ root = Path(__file__).resolve().parent.parent
333
+ action = yaml.safe_load((root / "action.yml").read_text())
334
+ assert action["runs"]["using"] == "composite"
335
+ assert set(action["inputs"]) == {"files", "version", "python-version"}
336
+ steps = " ".join(str(step.get("run", "")) for step in action["runs"]["steps"])
337
+ assert "model2data validate --glob" in steps
338
+ assert "--format github" in steps
339
+ hooks = yaml.safe_load((root / ".pre-commit-hooks.yaml").read_text())
340
+ assert hooks[0]["id"] == "model2data-validate"
341
+ assert hooks[0]["entry"] == "model2data validate"
342
+ assert hooks[0]["language"] == "python"
343
+ assert hooks[0]["pass_filenames"] is True
344
+
345
+
346
+ def test_validate_format_github_puts_the_line_on_a_value_issue(tmp_path, monkeypatch):
347
+ _write(
348
+ tmp_path,
349
+ "m.model2data.yml",
350
+ MODEL.replace("pk: true}\n email", "pk: maybe}\n email"),
351
+ )
352
+ monkeypatch.chdir(tmp_path)
353
+ result = runner.invoke(app, ["validate", "m.model2data.yml", "--format", "github"])
354
+ assert result.exit_code == 1
355
+ assert (
356
+ "::error file=m.model2data.yml,line=10::tables.customers.columns.id.pk: "
357
+ 'must be true or false, not the string "maybe"'
358
+ ) in result.output
359
+
360
+
361
+ def test_validate_summary_agrees_with_its_count(tmp_path):
362
+ good = _write(tmp_path, "good.model2data.yml", MODEL)
363
+ bad = _broken(tmp_path)
364
+ one_good = runner.invoke(app, ["validate", str(good), "--format", "github"])
365
+ assert "✅ 1 model file conforms." in one_good.output
366
+ one_bad = runner.invoke(app, ["validate", str(bad), "--format", "github"])
367
+ assert "❌ 1 of 1 model file does not conform." in one_bad.output
368
+ mixed = runner.invoke(app, ["validate", str(good), str(bad)])
369
+ assert "❌ 1 of 2 model files does not conform." in mixed.output
370
+ both = runner.invoke(app, ["validate", str(bad), str(_broken(tmp_path, "b2.model2data.yml"))])
371
+ assert "❌ 2 of 2 model files do not conform." in both.output
372
+
373
+
374
+ def test_action_uses_setup_python_v6():
375
+ import yaml
376
+
377
+ root = Path(__file__).resolve().parent.parent
378
+ action = yaml.safe_load((root / "action.yml").read_text())
379
+ uses = [step.get("uses") for step in action["runs"]["steps"]]
380
+ assert "actions/setup-python@v6" in uses
@@ -0,0 +1,87 @@
1
+ """Validation issues carry the line of the node their path points at."""
2
+
3
+ from model2data.model import Issue, validate
4
+ from model2data.model._yaml import _CoreLoader, _line_of_path, locate_issues
5
+
6
+ DOC = """\
7
+ model2data: 0.2.0
8
+ name: shop
9
+ tables:
10
+ raw.users:
11
+ columns:
12
+ id: {type: bigint, pk: true}
13
+ email: {type: email, bogus: 1}
14
+ orders:
15
+ columns:
16
+ id: {type: bigint, pk: maybe}
17
+ keys:
18
+ - unique: [id]
19
+ - unique: 5
20
+ """
21
+
22
+
23
+ def _line(path, text=DOC):
24
+ loader = _CoreLoader(text)
25
+ try:
26
+ return _line_of_path(loader.get_single_node(), path)
27
+ finally:
28
+ loader.dispose()
29
+
30
+
31
+ def test_a_simple_path_resolves_to_its_key_line():
32
+ assert _line("tables.orders.columns.id.pk") == 10
33
+ assert _line("name") == 2
34
+
35
+
36
+ def test_a_table_name_with_a_dot_resolves_by_longest_key():
37
+ assert _line("tables.raw.users.columns.email") == 7
38
+
39
+
40
+ def test_a_sequence_index_resolves_in_either_spelling():
41
+ assert _line("tables.orders.keys.1") == 13
42
+ assert _line("tables.orders.keys.[1]") == 13
43
+ assert _line("tables.orders.keys.1.unique") == 13
44
+
45
+
46
+ def test_an_unresolvable_tail_falls_back_to_the_deepest_node():
47
+ assert _line("tables.orders.columns.nope.type") == 9
48
+ assert _line("tables.orders.keys.9") == 11
49
+ assert _line("nothing.at.all") is None
50
+ assert _line("") is None
51
+
52
+
53
+ def test_validate_gives_issues_their_lines():
54
+ issues = validate(DOC)
55
+ by_path = {issue.path: issue for issue in issues}
56
+ assert by_path["tables.raw.users.columns.email.bogus"].line == 7
57
+ assert by_path["tables.orders.columns.id.pk"].line == 10
58
+ assert "(line 10)" in str(by_path["tables.orders.columns.id.pk"])
59
+
60
+
61
+ def test_json_issues_get_lines_too():
62
+ text = '{\n "model2data": "0.2.0",\n "name": "m",\n "tables": {\n "t": {"columns": {"id": {"type": "bigint", "pk": "maybe"}}}\n }\n}'
63
+ issues = validate(text)
64
+ assert [issue.line for issue in issues] == [5]
65
+
66
+
67
+ def test_dbml_issues_keep_no_line():
68
+ assert all(issue.line is None for issue in validate("Table t { id int }\n", format="dbml"))
69
+
70
+
71
+ def test_a_sequence_followed_by_a_name_stops_at_the_sequence():
72
+ # `keys` is a list: a path that names a key inside it cannot be followed.
73
+ assert _line("tables.orders.keys.unique") == 11
74
+
75
+
76
+ def test_a_path_past_a_plain_value_stops_at_that_value():
77
+ assert _line("name.first") == 2
78
+
79
+
80
+ def test_text_that_does_not_compose_leaves_the_issues_as_they_were():
81
+ issue = Issue("tables", "broken")
82
+ assert locate_issues("a: [unclosed", [issue]) == [issue]
83
+
84
+
85
+ def test_an_empty_document_leaves_the_issues_as_they_were():
86
+ issue = Issue("tables", "missing")
87
+ assert locate_issues("", [issue]) == [issue]
@@ -4,6 +4,7 @@ from __future__ import annotations
4
4
 
5
5
  import copy
6
6
  import json
7
+ from dataclasses import replace
7
8
  from pathlib import Path
8
9
 
9
10
  import pytest
@@ -350,7 +351,8 @@ def test_2_foreign_key_columns_pair_up():
350
351
  "are drawn from the ones it holds, but a parent of one row per value needs them to be "
351
352
  "its primary key, one of its keys, or a unique column",
352
353
  ]
353
- assert [str(issue) for issue in validate(json.dumps(document))][1:] == [
354
+ # `validate` on text adds the line each issue sits on; `from_dict` has no text.
355
+ assert [str(replace(issue, line=None)) for issue in validate(json.dumps(document))][1:] == [
354
356
  str(issue) for issue in raised.value.warnings
355
357
  ]
356
358
 
File without changes