model2data 1.20.0__tar.gz → 1.21.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {model2data-1.20.0/model2data.egg-info → model2data-1.21.1}/PKG-INFO +1 -1
- {model2data-1.20.0 → model2data-1.21.1}/README.md +5 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/cli.py +15 -6
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dbt/naming.py +19 -2
- model2data-1.21.1/model2data/dbt/templates/rollup_macros/model2data_rollup_tests.sql +58 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dbt/tests.py +59 -4
- {model2data-1.20.0 → model2data-1.21.1}/model2data/defects/checks.py +61 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/generate/core.py +114 -10
- {model2data-1.20.0 → model2data-1.21.1}/model2data/generate/days.py +23 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/generate/hints.py +7 -1
- {model2data-1.20.0 → model2data-1.21.1}/model2data/generate/parents.py +8 -1
- model2data-1.21.1/model2data/generate/rollups.py +698 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/generate/timeline.py +8 -2
- {model2data-1.20.0 → model2data-1.21.1}/model2data/guide/setup.md +2 -2
- {model2data-1.20.0 → model2data-1.21.1}/model2data/guide/triage.md +17 -5
- {model2data-1.20.0 → model2data-1.21.1}/model2data/guide/tune.md +36 -5
- {model2data-1.20.0 → model2data-1.21.1}/model2data/model/__init__.py +1 -1
- {model2data-1.20.0 → model2data-1.21.1}/model2data/model/dbml.py +6 -3
- {model2data-1.20.0 → model2data-1.21.1}/model2data/model/document.py +54 -2
- {model2data-1.20.0 → model2data-1.21.1}/model2data/model/types.py +1 -1
- {model2data-1.20.0 → model2data-1.21.1}/model2data/model/validate.py +68 -15
- {model2data-1.20.0 → model2data-1.21.1}/model2data/spec/PUBLISHING.md +13 -10
- {model2data-1.20.0 → model2data-1.21.1}/model2data/spec/README.md +79 -4
- {model2data-1.20.0 → model2data-1.21.1}/model2data/spec/model.schema.json +51 -5
- {model2data-1.20.0 → model2data-1.21.1/model2data.egg-info}/PKG-INFO +1 -1
- {model2data-1.20.0 → model2data-1.21.1}/model2data.egg-info/SOURCES.txt +3 -0
- {model2data-1.20.0 → model2data-1.21.1}/pyproject.toml +1 -1
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_dbt_integration.py +47 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_defects.py +2 -1
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_model_messages.py +4 -4
- model2data-1.21.1/tests/test_rollups.py +1055 -0
- {model2data-1.20.0 → model2data-1.21.1}/LICENSE +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/README_PYPI.md +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/LICENSE +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/VENDORED.md +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/_classes/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/_classes/base.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/_classes/column.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/_classes/enum.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/_classes/expression.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/_classes/index.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/_classes/note.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/_classes/project.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/_classes/reference.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/_classes/sticky_note.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/_classes/table.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/_classes/table_group.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/classes/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/constants.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/database.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/definitions/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/definitions/column.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/definitions/common.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/definitions/enum.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/definitions/generic.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/definitions/index.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/definitions/project.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/definitions/records.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/definitions/reference.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/definitions/sticky_note.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/definitions/table.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/definitions/table_group.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/definitions/table_partial.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/exceptions.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/parser/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/parser/blueprints.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/parser/parser.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/base.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/dbml/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/dbml/default/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/dbml/default/column.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/dbml/default/enum.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/dbml/default/expression.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/dbml/default/index.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/dbml/default/note.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/dbml/default/project.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/dbml/default/reference.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/dbml/default/renderer.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/dbml/default/sticky_note.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/dbml/default/table.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/dbml/default/table_group.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/dbml/default/utils.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/sql/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/sql/default/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/sql/default/column.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/sql/default/enum.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/sql/default/expression.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/sql/default/index.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/sql/default/note.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/sql/default/reference.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/sql/default/renderer.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/sql/default/table.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/renderer/sql/default/utils.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/_vendor/pydbml/tools.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dashboards/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dashboards/check.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dashboards/reader.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dashboards/types.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dashboards/writer.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dbt/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dbt/hint_tests.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dbt/project.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dbt/templates/dbt_project.yml.jinja +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dbt/templates/hint_macros/model2data_hint_tests.sql +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dbt/templates/hint_macros/model2data_parent_tests.sql +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dbt/templates/hint_macros/model2data_when_tests.sql +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dbt/templates/history_macros/model2data_history_tests.sql +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dbt/templates/macros/generate_schema_name.sql +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/dbt/templates/profiles.yml.jinja +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/defects/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/defects/apply.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/defects/presets.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/defects/report.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/generate/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/generate/faker.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/generate/history.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/generate/kinds.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/generate/options.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/generate/relationships.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/generate/when.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/guide/dashboards.md +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/check.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/columns.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/dbt.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/expression.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/graph.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/infer.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/lightdash.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/ossie.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/reader.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/semantic.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/slices.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/sql.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/types.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/metrics/values.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/model/_yaml.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/model/dump.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/model/engine.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/model/errors.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/model/reader.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/output.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/parse/__init__.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/parse/dbml.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/spec/dashboards/README.md +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/spec/dashboards/dashboard.schema.json +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/spec/examples/coffee_webshop.metrics.yml +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/spec/examples/coffee_webshop.model2data.yml +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/spec/examples/coffee_webshop_sales.dashboard.yml +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/spec/metrics/README.md +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/spec/metrics/metrics.schema.json +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data/utils.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data.egg-info/dependency_links.txt +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data.egg-info/entry_points.txt +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data.egg-info/requires.txt +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/model2data.egg-info/top_level.txt +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/setup.cfg +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_as_of_anchor.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_cli.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_cli_model.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_column_time_hints.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_coverage_gaps.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_dashboards.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_dashboards_cli.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_days.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_dbml_parser.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_dbml_parser_fuzz.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_dbt_naming.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_dbt_project.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_dbt_tests.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_defects_dbt.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_defects_replay.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_defects_report.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_defects_review.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_determinism.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_distributions.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_examples_model.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_faker_name_inference.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_fast_paths.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_generation.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_guide.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_hint_tests.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_history.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_issue_lines.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_lone_country.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_metrics_cli.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_metrics_dbt.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_metrics_expression.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_metrics_lightdash.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_metrics_ossie.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_metrics_prepared.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_metrics_reader.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_metrics_semantic.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_metrics_slices.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_metrics_spec.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_metrics_values.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_model.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_model_dbml.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_model_edges.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_model_vocabulary.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_options.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_parent_dates.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_release_stress.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_reproducibility.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_row_identity.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_shaping.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_spec_alignment.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_spec_schema.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_table_seeds.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_timeline.py +0 -0
- {model2data-1.20.0 → model2data-1.21.1}/tests/test_when.py +0 -0
|
@@ -26,6 +26,11 @@ cd dbt_ecommerce && dbt build # PA
|
|
|
26
26
|
> is this engine in the browser: draw the schema, see every column's values before you generate,
|
|
27
27
|
> shape the data on a sample of your own, and export a dbt project with a star schema or Data
|
|
28
28
|
> Vault on top. [Start free, no install →](https://studio.jbanalytica.com)
|
|
29
|
+
>
|
|
30
|
+
> **Or ask Claude.** Add `https://studio.jbanalytica.com/mcp` as a connector in Claude Code,
|
|
31
|
+
> Claude Desktop or claude.ai, sign in, and Claude builds the model, the data, metrics with known
|
|
32
|
+
> answers and a dashboard from one sentence. Works on the free plan.
|
|
33
|
+
> [](https://glama.ai/mcp/connectors/io.github.JB-Analytica/model2data-studio)
|
|
29
34
|
|
|
30
35
|
---
|
|
31
36
|
|
|
@@ -33,6 +33,7 @@ from model2data.generate.core import (
|
|
|
33
33
|
generate_data_from_dbml,
|
|
34
34
|
get_capped_tables,
|
|
35
35
|
get_cyclic_tables,
|
|
36
|
+
get_uncovered_parents,
|
|
36
37
|
get_unresolved_composite_keys,
|
|
37
38
|
)
|
|
38
39
|
from model2data.generate.days import (
|
|
@@ -187,7 +188,7 @@ app = typer.Typer(
|
|
|
187
188
|
cls=_GenerateByDefault,
|
|
188
189
|
help=(
|
|
189
190
|
"model2data: Generate analytics-ready datasets from a data model.\n\n"
|
|
190
|
-
"Given a model -- a .model2data.yml document (spec 0.
|
|
191
|
+
"Given a model -- a .model2data.yml document (spec 0.6.0), the same as JSON,\n"
|
|
191
192
|
"or a DBML file -- this tool produces:\n"
|
|
192
193
|
"• Synthetic but realistic data\n"
|
|
193
194
|
"• A runnable dbt project scaffold\n"
|
|
@@ -777,6 +778,7 @@ def main(
|
|
|
777
778
|
unresolved_keys = get_unresolved_composite_keys()
|
|
778
779
|
duplicate_unique = get_duplicate_unique_columns()
|
|
779
780
|
capped_tables = get_capped_tables()
|
|
781
|
+
uncovered = get_uncovered_parents()
|
|
780
782
|
|
|
781
783
|
typer.echo("\n📊 Summary")
|
|
782
784
|
typer.echo(f" Tables generated: {len(generated_tables)}")
|
|
@@ -822,6 +824,13 @@ def main(
|
|
|
822
824
|
)
|
|
823
825
|
for label in capped_tables:
|
|
824
826
|
typer.echo(f" - {label}")
|
|
827
|
+
if uncovered:
|
|
828
|
+
typer.echo(
|
|
829
|
+
" ⚠️ Foreign keys with too few rows for their at_least (raise the table's rows "
|
|
830
|
+
"to cover every parent):"
|
|
831
|
+
)
|
|
832
|
+
for label in uncovered:
|
|
833
|
+
typer.echo(f" - {label}")
|
|
825
834
|
|
|
826
835
|
if metrics_count is not None:
|
|
827
836
|
typer.echo(
|
|
@@ -849,10 +858,10 @@ def main(
|
|
|
849
858
|
|
|
850
859
|
|
|
851
860
|
def _spec_of(file: Path) -> str:
|
|
852
|
-
"""The spec a conforming model is written against: 0.5.0, 0.4.0, 0.3.0, or 0.2.0
|
|
853
|
-
without `when
|
|
861
|
+
"""The spec a conforming model is written against: 0.6.0, 0.5.0, 0.4.0, 0.3.0, or 0.2.0
|
|
862
|
+
(DBML without `when`, a 0.5.0 `after`, `rollup` or `at_least` included)."""
|
|
854
863
|
version = str(load(file).version)
|
|
855
|
-
for minor in ("0.5", "0.4", "0.3"):
|
|
864
|
+
for minor in ("0.6", "0.5", "0.4", "0.3"):
|
|
856
865
|
if version.startswith(minor):
|
|
857
866
|
return f"{minor}.0"
|
|
858
867
|
return "0.2.0"
|
|
@@ -948,7 +957,7 @@ def validate_command(
|
|
|
948
957
|
),
|
|
949
958
|
),
|
|
950
959
|
):
|
|
951
|
-
"""Check that models conform to spec 0.
|
|
960
|
+
"""Check that models conform to spec 0.6.0 (or 0.2.x-0.5.x), printing every issue with its path.
|
|
952
961
|
|
|
953
962
|
Takes one or more files and/or --glob patterns. A metrics file (*.metrics.yml) is checked
|
|
954
963
|
against metrics spec 0.1.0 and against its model; a dashboard (*.dashboard.yml) against
|
|
@@ -1057,7 +1066,7 @@ def convert_command(
|
|
|
1057
1066
|
):
|
|
1058
1067
|
"""Convert a model -- DBML, typically -- to a .model2data.yml document (spec 0.2.0, or
|
|
1059
1068
|
0.3.0 when it has defects, 0.4.0 when it has `when`, 0.5.0 when an `after` is a list or
|
|
1060
|
-
names a parent's column)."""
|
|
1069
|
+
names a parent's column, 0.6.0 when it has `rollup` or `at_least`)."""
|
|
1061
1070
|
try:
|
|
1062
1071
|
model = load(file)
|
|
1063
1072
|
except ModelError as error:
|
|
@@ -10,7 +10,7 @@ import re
|
|
|
10
10
|
from collections.abc import Iterable, Mapping
|
|
11
11
|
from dataclasses import replace
|
|
12
12
|
|
|
13
|
-
from model2data.parse.dbml import TableDef
|
|
13
|
+
from model2data.parse.dbml import ColumnDef, TableDef
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
def dbt_identifier(key: str) -> str:
|
|
@@ -49,7 +49,14 @@ def for_dbt(
|
|
|
49
49
|
tables: Mapping[str, TableDef], refs: list[dict], names: Mapping[str, str]
|
|
50
50
|
) -> tuple[dict[str, TableDef], list[dict]]:
|
|
51
51
|
"""`(tables, refs)` renamed to their dbt names, as `generate_dbt_yml` takes them."""
|
|
52
|
-
dbt_tables = {
|
|
52
|
+
dbt_tables = {
|
|
53
|
+
names[key]: replace(
|
|
54
|
+
table,
|
|
55
|
+
name=names[key],
|
|
56
|
+
columns=[_rollup_renamed(column, names) for column in table.columns],
|
|
57
|
+
)
|
|
58
|
+
for key, table in tables.items()
|
|
59
|
+
}
|
|
53
60
|
dbt_refs = [
|
|
54
61
|
{
|
|
55
62
|
**ref,
|
|
@@ -59,3 +66,13 @@ def for_dbt(
|
|
|
59
66
|
for ref in refs
|
|
60
67
|
]
|
|
61
68
|
return dbt_tables, dbt_refs
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _rollup_renamed(column: ColumnDef, names: Mapping[str, str]) -> ColumnDef:
|
|
72
|
+
"""`column` with its `rollup`'s child table (`of`) under its dbt name."""
|
|
73
|
+
note = column.note if isinstance(column.note, dict) else None
|
|
74
|
+
spec = note.get("rollup") if note else None
|
|
75
|
+
child = spec.get("of") if isinstance(spec, dict) else None
|
|
76
|
+
if not isinstance(child, str) or child not in names or names[child] == child:
|
|
77
|
+
return column
|
|
78
|
+
return replace(column, note={**note, "rollup": {**spec, "of": names[child]}})
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{#-
|
|
2
|
+
The generic test model2data writes for a `rollup` (`rollup: {of: order_items,
|
|
3
|
+
sum: [quantity, unit_price]}` on orders.total_amount): the column holds the
|
|
4
|
+
aggregate of the child rows whose foreign key points at its row. Written
|
|
5
|
+
only by a model that has one. Plain ANSI SQL, runs on DuckDB and Postgres.
|
|
6
|
+
Returns the rows that break it.
|
|
7
|
+
|
|
8
|
+
`column_name` arrives already quoted by the schema YAML; `child` is the child
|
|
9
|
+
model (`ref('stg_order_items')`), `foreign_key` its column onto this model's
|
|
10
|
+
`field`. `aggregate` is sum, count, count_distinct, min or max, over the
|
|
11
|
+
child's `columns` (a sum multiplies them on each child row). A row with no
|
|
12
|
+
child rows expects 0 for a sum or a count and null for min and max. A sum is
|
|
13
|
+
rounded once per row by the generator, so it passes within `tolerance` (half
|
|
14
|
+
a unit of its last decimal place), plus 1e-9 for floating-point noise; every
|
|
15
|
+
other aggregate compares exactly.
|
|
16
|
+
-#}
|
|
17
|
+
|
|
18
|
+
{% test model2data_rollup(model, column_name, child, foreign_key, field, aggregate, columns=none, tolerance=0) %}
|
|
19
|
+
{%- set quoted = [] -%}
|
|
20
|
+
{%- for name in (columns or []) -%}
|
|
21
|
+
{%- do quoted.append(adapter.quote(name)) -%}
|
|
22
|
+
{%- endfor -%}
|
|
23
|
+
{%- if aggregate == 'count' -%}
|
|
24
|
+
{%- set expression = 'count(*)' -%}
|
|
25
|
+
{%- elif aggregate == 'count_distinct' -%}
|
|
26
|
+
{%- set expression = 'count(distinct ' ~ quoted[0] ~ ')' -%}
|
|
27
|
+
{%- elif aggregate == 'sum' -%}
|
|
28
|
+
{%- set expression = 'sum(' ~ (quoted | join(' * ')) ~ ')' -%}
|
|
29
|
+
{%- else -%}
|
|
30
|
+
{%- set expression = aggregate ~ '(' ~ quoted[0] ~ ')' -%}
|
|
31
|
+
{%- endif %}
|
|
32
|
+
with rolled as (
|
|
33
|
+
select
|
|
34
|
+
{{ adapter.quote(foreign_key) }} as parent_key,
|
|
35
|
+
{{ expression }} as expected
|
|
36
|
+
from {{ child }}
|
|
37
|
+
where {{ adapter.quote(foreign_key) }} is not null
|
|
38
|
+
group by {{ adapter.quote(foreign_key) }}
|
|
39
|
+
)
|
|
40
|
+
select
|
|
41
|
+
parent.{{ column_name }} as value,
|
|
42
|
+
rolled.expected
|
|
43
|
+
from {{ model }} as parent
|
|
44
|
+
left join rolled
|
|
45
|
+
on parent.{{ adapter.quote(field) }} = rolled.parent_key
|
|
46
|
+
where
|
|
47
|
+
{%- if aggregate in ('min', 'max') %}
|
|
48
|
+
not (parent.{{ column_name }} is null and rolled.expected is null)
|
|
49
|
+
and (
|
|
50
|
+
parent.{{ column_name }} is null
|
|
51
|
+
or rolled.expected is null
|
|
52
|
+
or parent.{{ column_name }} <> rolled.expected
|
|
53
|
+
)
|
|
54
|
+
{%- else %}
|
|
55
|
+
parent.{{ column_name }} is null
|
|
56
|
+
or abs(parent.{{ column_name }} - coalesce(rolled.expected, 0)) > {{ tolerance }} + 0.000000001
|
|
57
|
+
{%- endif %}
|
|
58
|
+
{% endtest %}
|
|
@@ -11,6 +11,7 @@ import yaml
|
|
|
11
11
|
from model2data.dbt.hint_tests import DEFAULT_TOLERANCE, HintTest, hint_tests_for, write_hint_macros
|
|
12
12
|
from model2data.generate.faker import is_free_text_type
|
|
13
13
|
from model2data.generate.relationships import classify_refs
|
|
14
|
+
from model2data.generate.rollups import Rollup, has_rollups, rollups_of
|
|
14
15
|
|
|
15
16
|
# dbt nests a generic test's parameters under `arguments:`. Passing them as
|
|
16
17
|
# bare keys still works but is deprecated, and warns on every single run --
|
|
@@ -75,6 +76,9 @@ def generate_dbt_yml(
|
|
|
75
76
|
if any(_history_entries(table) for table in tables.values()):
|
|
76
77
|
write_history_macros(dest)
|
|
77
78
|
fk_map = _fk_map(tables, refs)
|
|
79
|
+
rolled = _rollups(tables, refs)
|
|
80
|
+
if rolled:
|
|
81
|
+
write_rollup_macros(dest)
|
|
78
82
|
|
|
79
83
|
# -------------------------
|
|
80
84
|
# Generate individual staging model YAMLs
|
|
@@ -84,7 +88,7 @@ def generate_dbt_yml(
|
|
|
84
88
|
model_columns = []
|
|
85
89
|
|
|
86
90
|
for col in table.columns:
|
|
87
|
-
tests = _column_entries(table, col, fk_map, hinted, hint_tests)
|
|
91
|
+
tests = _column_entries(table, col, fk_map, hinted, hint_tests, rolled)
|
|
88
92
|
col_doc: dict[str, Any] = {"name": _dbt_column_ref(col.name)}
|
|
89
93
|
description = getattr(col, "description", None)
|
|
90
94
|
if description:
|
|
@@ -157,6 +161,7 @@ def _column_entries(
|
|
|
157
161
|
fk_map: dict[tuple[str, str], list[dict]],
|
|
158
162
|
hinted: dict[tuple[str, Optional[str]], list[HintTest]],
|
|
159
163
|
hint_tests: str,
|
|
164
|
+
rolled: Optional[dict[tuple[str, str], Rollup]] = None,
|
|
160
165
|
) -> list[Union[str, dict[str, dict[str, Any]]]]:
|
|
161
166
|
"""The `tests:` entries of one column of a staging model's schema YAML."""
|
|
162
167
|
tests: list[Union[str, dict[str, dict[str, Any]]]] = []
|
|
@@ -181,9 +186,54 @@ def _column_entries(
|
|
|
181
186
|
)
|
|
182
187
|
|
|
183
188
|
tests.extend(t.to_dbt(hint_tests) for t in hinted.get((table.name, col.name), []))
|
|
189
|
+
rollup = (rolled or {}).get((table.name, col.name))
|
|
190
|
+
if rollup is not None:
|
|
191
|
+
tests.append(_generic_test("model2data_rollup", rollup_arguments(rollup)))
|
|
184
192
|
return tests
|
|
185
193
|
|
|
186
194
|
|
|
195
|
+
def _rollups(tables: dict, refs: list[dict]) -> dict[tuple[str, str], Rollup]:
|
|
196
|
+
"""The model's rollups by (table, column); `for_dbt` names their child tables for dbt."""
|
|
197
|
+
if not has_rollups(tables):
|
|
198
|
+
return {}
|
|
199
|
+
return {(r.table, r.column): r for r in rollups_of(tables, refs)}
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def rollup_arguments(rollup: Rollup) -> dict[str, Any]:
|
|
203
|
+
"""The `model2data_rollup` test's arguments for `rollup`.
|
|
204
|
+
|
|
205
|
+
A sum compares within half a unit of its last decimal place (`tolerance`),
|
|
206
|
+
since it is rounded once per parent; every other aggregate exactly.
|
|
207
|
+
"""
|
|
208
|
+
arguments: dict[str, Any] = {
|
|
209
|
+
"child": f"ref('stg_{rollup.child}')",
|
|
210
|
+
"foreign_key": rollup.fk,
|
|
211
|
+
"field": rollup.parent_key,
|
|
212
|
+
"aggregate": rollup.aggregate,
|
|
213
|
+
}
|
|
214
|
+
if rollup.columns:
|
|
215
|
+
arguments["columns"] = list(rollup.columns)
|
|
216
|
+
if rollup.aggregate == "sum":
|
|
217
|
+
arguments["tolerance"] = 0.5 / 10**rollup.scale if rollup.scale else 0
|
|
218
|
+
return arguments
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
ROLLUP_MACROS_FILE = "model2data_rollup_tests.sql"
|
|
222
|
+
_ROLLUP_MACROS = Path(__file__).parent / "templates" / "rollup_macros" / ROLLUP_MACROS_FILE
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def write_rollup_macros(dest: Path) -> Path:
|
|
226
|
+
"""Write the rollup test into `dest/macros/`, and return the file's path.
|
|
227
|
+
|
|
228
|
+
`generate_dbt_yml` writes it when a column has a `rollup`, and only then.
|
|
229
|
+
"""
|
|
230
|
+
macros = dest / "macros"
|
|
231
|
+
macros.mkdir(parents=True, exist_ok=True)
|
|
232
|
+
target = macros / ROLLUP_MACROS_FILE
|
|
233
|
+
target.write_text(_ROLLUP_MACROS.read_text())
|
|
234
|
+
return target
|
|
235
|
+
|
|
236
|
+
|
|
187
237
|
@dataclass(frozen=True)
|
|
188
238
|
class DbtTest:
|
|
189
239
|
"""One data test `generate_dbt_yml` writes, under the name dbt gives it.
|
|
@@ -195,8 +245,9 @@ class DbtTest:
|
|
|
195
245
|
`relationships`, a `model2data_*` hint test) or `unique_combination` for the
|
|
196
246
|
singular test of a composite key. `arguments` are the test's parameters,
|
|
197
247
|
`severity` is `error`, or the hint tests' severity, and `parent` is the
|
|
198
|
-
`(table, column)` a `relationships` test looks values up in,
|
|
199
|
-
key a `model2data_not_before_parent` test joins on
|
|
248
|
+
`(table, column)` a `relationships` test looks values up in, the parent
|
|
249
|
+
key a `model2data_not_before_parent` test joins on, or the child table and
|
|
250
|
+
foreign key a `model2data_rollup` test aggregates.
|
|
200
251
|
"""
|
|
201
252
|
|
|
202
253
|
name: str
|
|
@@ -223,6 +274,7 @@ def dbt_tests(
|
|
|
223
274
|
"""
|
|
224
275
|
hinted = _hinted(tables, hint_tests, test_tolerance, refs)
|
|
225
276
|
fk_map = _fk_map(tables, refs)
|
|
277
|
+
rolled = _rollups(tables, refs)
|
|
226
278
|
found: list[DbtTest] = []
|
|
227
279
|
for table in tables.values():
|
|
228
280
|
stg_name = f"stg_{table.name}"
|
|
@@ -232,7 +284,7 @@ def dbt_tests(
|
|
|
232
284
|
hint_parents = iter(
|
|
233
285
|
t.parent for t in hinted.get((table.name, col.name), []) if t.parent is not None
|
|
234
286
|
)
|
|
235
|
-
for entry in _column_entries(table, col, fk_map, hinted, hint_tests):
|
|
287
|
+
for entry in _column_entries(table, col, fk_map, hinted, hint_tests, rolled):
|
|
236
288
|
test_type, body = _entry_parts(entry)
|
|
237
289
|
arguments = dict(body.get("arguments") or {})
|
|
238
290
|
parent = None
|
|
@@ -241,6 +293,9 @@ def dbt_tests(
|
|
|
241
293
|
parent = (ref["target_table"], ref["target_column"])
|
|
242
294
|
elif test_type == "model2data_not_before_parent":
|
|
243
295
|
parent = next(hint_parents)
|
|
296
|
+
elif test_type == "model2data_rollup":
|
|
297
|
+
rollup = rolled[(table.name, col.name)]
|
|
298
|
+
parent = (rollup.child, rollup.fk)
|
|
244
299
|
found.append(
|
|
245
300
|
DbtTest(
|
|
246
301
|
name=generic_test_name(
|
|
@@ -23,6 +23,10 @@ SQL reads:
|
|
|
23
23
|
in any case, a number as a number).
|
|
24
24
|
- `model2data_unique_combination` and a composite key's `unique_combination`: a
|
|
25
25
|
combination of values on two rows, nulls grouping together as in `group by`.
|
|
26
|
+
- `model2data_rollup`: as its macro in
|
|
27
|
+
`dbt/templates/rollup_macros/model2data_rollup_tests.sql`: a row whose value is
|
|
28
|
+
not the aggregate of the child rows its key matches (0 for a sum or count with
|
|
29
|
+
none, null for min and max), a sum within its `tolerance`.
|
|
26
30
|
- A history table's `model2data_one_current_row` (a key with other than one
|
|
27
31
|
current version) and `model2data_no_overlapping_ranges` (a version still
|
|
28
32
|
valid when the next of its key begins).
|
|
@@ -462,6 +466,62 @@ def _no_overlapping_ranges(
|
|
|
462
466
|
return False
|
|
463
467
|
|
|
464
468
|
|
|
469
|
+
def _rollup(test: DbtTest, frame: pd.DataFrame, seeded: Mapping, cache: Columns) -> bool:
|
|
470
|
+
assert test.parent is not None # `dbt_tests` names the child of every such test
|
|
471
|
+
child = seeded.get(test.parent[0])
|
|
472
|
+
if child is None:
|
|
473
|
+
return False
|
|
474
|
+
arguments = test.arguments
|
|
475
|
+
aggregate = arguments["aggregate"]
|
|
476
|
+
columns = list(arguments.get("columns") or [])
|
|
477
|
+
keys = cache.numbers(child, arguments["foreign_key"])
|
|
478
|
+
groups: dict[Any, list[Any]] = {}
|
|
479
|
+
for row, key in enumerate(keys):
|
|
480
|
+
if key is None:
|
|
481
|
+
continue
|
|
482
|
+
groups.setdefault(key, []).append(row)
|
|
483
|
+
numbers = {name: cache.numbers(child, name) for name in columns}
|
|
484
|
+
|
|
485
|
+
def expected(rows: list[int]) -> Any:
|
|
486
|
+
if aggregate == "count":
|
|
487
|
+
return float(len(rows))
|
|
488
|
+
if aggregate == "sum":
|
|
489
|
+
total = 0.0
|
|
490
|
+
for row in rows:
|
|
491
|
+
factors = [numbers[name][row] for name in columns]
|
|
492
|
+
if all(isinstance(f, float) for f in factors):
|
|
493
|
+
total += float(np.prod(factors))
|
|
494
|
+
return total
|
|
495
|
+
values = [numbers[columns[0]][row] for row in rows]
|
|
496
|
+
values = [value for value in values if value is not None]
|
|
497
|
+
if aggregate == "count_distinct":
|
|
498
|
+
return float(len(set(values)))
|
|
499
|
+
if not values:
|
|
500
|
+
return None
|
|
501
|
+
if not all(isinstance(value, float) for value in values):
|
|
502
|
+
values = list(pd.to_datetime(pd.Series(values, dtype=object), format="mixed"))
|
|
503
|
+
return min(values) if aggregate == "min" else max(values)
|
|
504
|
+
|
|
505
|
+
tolerance = float(arguments.get("tolerance") or 0) + 1e-9
|
|
506
|
+
own = cache.numbers(frame, str(test.column))
|
|
507
|
+
parent_keys = cache.numbers(frame, arguments["field"])
|
|
508
|
+
for value, key in zip(own, parent_keys, strict=True):
|
|
509
|
+
rows = groups.get(key, [])
|
|
510
|
+
want = expected(rows) if rows else (None if aggregate in ("min", "max") else 0.0)
|
|
511
|
+
if aggregate in ("min", "max"):
|
|
512
|
+
if value is None and want is None:
|
|
513
|
+
continue
|
|
514
|
+
if value is None or want is None:
|
|
515
|
+
return True
|
|
516
|
+
if not isinstance(value, float):
|
|
517
|
+
value = pd.to_datetime(value, format="mixed")
|
|
518
|
+
if value != want:
|
|
519
|
+
return True
|
|
520
|
+
elif not isinstance(value, float) or abs(value - want) > tolerance:
|
|
521
|
+
return True
|
|
522
|
+
return False
|
|
523
|
+
|
|
524
|
+
|
|
465
525
|
_CHECKS: dict[str, Callable[[DbtTest, pd.DataFrame, Mapping, Columns], bool]] = {
|
|
466
526
|
"not_null": _not_null,
|
|
467
527
|
"unique": _unique,
|
|
@@ -478,4 +538,5 @@ _CHECKS: dict[str, Callable[[DbtTest, pd.DataFrame, Mapping, Columns], bool]] =
|
|
|
478
538
|
"unique_combination": _unique_combination,
|
|
479
539
|
"model2data_one_current_row": _one_current_row,
|
|
480
540
|
"model2data_no_overlapping_ranges": _no_overlapping_ranges,
|
|
541
|
+
"model2data_rollup": _rollup,
|
|
481
542
|
}
|
|
@@ -5,7 +5,7 @@ import random
|
|
|
5
5
|
from collections import defaultdict, deque
|
|
6
6
|
from collections.abc import Mapping
|
|
7
7
|
from datetime import datetime
|
|
8
|
-
from typing import Optional
|
|
8
|
+
from typing import Any, Optional
|
|
9
9
|
|
|
10
10
|
import pandas as pd
|
|
11
11
|
from faker import Faker
|
|
@@ -27,6 +27,14 @@ from model2data.generate.relationships import (
|
|
|
27
27
|
build_fk_lookup,
|
|
28
28
|
classify_refs,
|
|
29
29
|
)
|
|
30
|
+
from model2data.generate.rollups import (
|
|
31
|
+
Coverage,
|
|
32
|
+
apply_rollups,
|
|
33
|
+
at_least_of,
|
|
34
|
+
cover_parents,
|
|
35
|
+
has_rollups,
|
|
36
|
+
protected_rows,
|
|
37
|
+
)
|
|
30
38
|
from model2data.generate.timeline import _resolve_anchor, order_row_times
|
|
31
39
|
from model2data.generate.when import SeedFor, apply_when
|
|
32
40
|
from model2data.parse.dbml import ColumnDef, TableDef
|
|
@@ -77,8 +85,9 @@ _capped_tables: list[str] = []
|
|
|
77
85
|
|
|
78
86
|
|
|
79
87
|
def reset_capped_tables() -> None:
|
|
80
|
-
"""Clear the record of tables cut to fit a unique reference."""
|
|
88
|
+
"""Clear the record of tables cut to fit a unique reference, and of parents left uncovered."""
|
|
81
89
|
_capped_tables.clear()
|
|
90
|
+
_uncovered_parents.clear()
|
|
82
91
|
|
|
83
92
|
|
|
84
93
|
def get_capped_tables() -> list[str]:
|
|
@@ -86,6 +95,17 @@ def get_capped_tables() -> list[str]:
|
|
|
86
95
|
return list(_capped_tables)
|
|
87
96
|
|
|
88
97
|
|
|
98
|
+
# Foreign keys whose `at_least` the most recent run could not give every parent:
|
|
99
|
+
# the table has fewer rows than at_least x parents. Not an error; reported beside
|
|
100
|
+
# the capped tables.
|
|
101
|
+
_uncovered_parents: list[str] = []
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def get_uncovered_parents() -> list[str]:
|
|
105
|
+
"""Return "table: N rows cover C of P parent; at_least ... needs M" labels."""
|
|
106
|
+
return list(_uncovered_parents)
|
|
107
|
+
|
|
108
|
+
|
|
89
109
|
def _unique_reference_cap(
|
|
90
110
|
table_def: TableDef,
|
|
91
111
|
table_name: str,
|
|
@@ -275,6 +295,9 @@ def generate_data_from_dbml(
|
|
|
275
295
|
# `postcode` beside it to keep coherent with. See
|
|
276
296
|
# _lone_country_columns.
|
|
277
297
|
lone_country_columns = _lone_country_columns(table_def)
|
|
298
|
+
# The rows an `at_least` gave a parent, by foreign-key column: nothing
|
|
299
|
+
# after the first pass moves them to another parent (see generate.rollups).
|
|
300
|
+
coverage: dict[str, Coverage] = {}
|
|
278
301
|
|
|
279
302
|
# -----------------------
|
|
280
303
|
# First pass: columns + FKs
|
|
@@ -295,9 +318,8 @@ def generate_data_from_dbml(
|
|
|
295
318
|
# generated test could fail non-deterministically on a chance
|
|
296
319
|
# collision (e.g. a "unique" promo_code or VIN column).
|
|
297
320
|
ensure_unique = "pk" in column.settings or "unique" in column.settings
|
|
298
|
-
|
|
321
|
+
draw = _drawer(
|
|
299
322
|
column=column,
|
|
300
|
-
row_count=row_count,
|
|
301
323
|
fk_series=fk_series,
|
|
302
324
|
ensure_unique=ensure_unique,
|
|
303
325
|
force_not_null=column.name in composite_pk_columns,
|
|
@@ -307,12 +329,16 @@ def generate_data_from_dbml(
|
|
|
307
329
|
skew=skew,
|
|
308
330
|
lone_country=column.name in lone_country_columns,
|
|
309
331
|
)
|
|
332
|
+
data[column.name] = _covered(
|
|
333
|
+
draw, column, table_name, fk_target, fk_series, row_count, coverage
|
|
334
|
+
)
|
|
310
335
|
|
|
311
336
|
df = pd.DataFrame(data)
|
|
312
337
|
# A cross-table `after` keeps a row's date on or after the parent row
|
|
313
338
|
# it points at: before the row's own later dates are ordered, so they
|
|
314
339
|
# follow it wherever it ends up (see generate.parents).
|
|
315
340
|
table_rules = rules.get(table_name, [])
|
|
341
|
+
keep = protected_rows(coverage, row_count) if coverage else None
|
|
316
342
|
floors = {
|
|
317
343
|
rule.column: follow_parents(
|
|
318
344
|
df,
|
|
@@ -321,6 +347,7 @@ def generate_data_from_dbml(
|
|
|
321
347
|
generated,
|
|
322
348
|
as_of=as_of,
|
|
323
349
|
time_profile=profile,
|
|
350
|
+
keep=keep,
|
|
324
351
|
)[0]
|
|
325
352
|
for rule in table_rules
|
|
326
353
|
}
|
|
@@ -338,6 +365,7 @@ def generate_data_from_dbml(
|
|
|
338
365
|
as_of=as_of,
|
|
339
366
|
time_profile=profile,
|
|
340
367
|
skew=skew,
|
|
368
|
+
coverage=coverage,
|
|
341
369
|
)
|
|
342
370
|
df = _deduplicate_composite_keys(
|
|
343
371
|
df,
|
|
@@ -348,6 +376,7 @@ def generate_data_from_dbml(
|
|
|
348
376
|
as_of=as_of,
|
|
349
377
|
time_profile=profile,
|
|
350
378
|
skew=skew,
|
|
379
|
+
keep=protected_rows(coverage, row_count) if coverage else None,
|
|
351
380
|
)
|
|
352
381
|
|
|
353
382
|
# -----------------------------------------------------
|
|
@@ -391,23 +420,73 @@ def generate_data_from_dbml(
|
|
|
391
420
|
Faker.seed(stream_seed)
|
|
392
421
|
reset_row_pools()
|
|
393
422
|
df = generated[table_name]
|
|
394
|
-
|
|
423
|
+
fk_series = generated[parent_table][parent_column]
|
|
424
|
+
draw = _drawer(
|
|
395
425
|
column=column,
|
|
396
|
-
|
|
397
|
-
fk_series=generated[parent_table][parent_column],
|
|
426
|
+
fk_series=fk_series,
|
|
398
427
|
ensure_unique="unique" in column.settings,
|
|
399
428
|
table_name=table_name,
|
|
400
429
|
as_of=as_of,
|
|
401
430
|
time_profile=profile,
|
|
402
431
|
skew=skew,
|
|
403
432
|
)
|
|
433
|
+
df[column.name] = _covered(
|
|
434
|
+
draw, column, table_name, (parent_table, parent_column), fk_series, len(df), {}
|
|
435
|
+
)
|
|
404
436
|
df = _mirror_attributes(df, table_name, attribute_refs, fk_refs, generated)
|
|
405
437
|
generated[table_name] = _coerce_integer_dtypes(df, table_def)
|
|
406
438
|
release_row_pools(table_name)
|
|
407
439
|
|
|
440
|
+
# ---------------------------------------------------------
|
|
441
|
+
# Rollups, once every table holds its final rows: no draws, so nothing
|
|
442
|
+
# else changes (see generate.rollups).
|
|
443
|
+
# ---------------------------------------------------------
|
|
444
|
+
if has_rollups(tables):
|
|
445
|
+
for table_name in apply_rollups(generated, tables, refs):
|
|
446
|
+
generated[table_name] = _coerce_integer_dtypes(
|
|
447
|
+
generated[table_name], tables[table_name]
|
|
448
|
+
)
|
|
449
|
+
|
|
408
450
|
return generated
|
|
409
451
|
|
|
410
452
|
|
|
453
|
+
def _drawer(**arguments):
|
|
454
|
+
"""`generate_column_values` with every argument but the row count bound."""
|
|
455
|
+
|
|
456
|
+
def draw(row_count: int) -> list:
|
|
457
|
+
return generate_column_values(row_count=row_count, **arguments)
|
|
458
|
+
|
|
459
|
+
return draw
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _covered(
|
|
463
|
+
draw,
|
|
464
|
+
column: ColumnDef,
|
|
465
|
+
table_name: str,
|
|
466
|
+
fk_target: Optional[tuple[str, str]],
|
|
467
|
+
fk_series: Optional[pd.Series],
|
|
468
|
+
row_count: int,
|
|
469
|
+
coverage: dict[str, Coverage],
|
|
470
|
+
) -> list:
|
|
471
|
+
"""The column's values: `draw(row_count)`, or with its `at_least` parents covered.
|
|
472
|
+
|
|
473
|
+
A column without `at_least`, or with no parent values to cover, draws
|
|
474
|
+
exactly as it always has. A shortfall is reported (`get_uncovered_parents`).
|
|
475
|
+
"""
|
|
476
|
+
at_least = at_least_of(column)
|
|
477
|
+
if not at_least or fk_series is None or not bool(fk_series.notna().any()):
|
|
478
|
+
return draw(row_count)
|
|
479
|
+
values, cover = cover_parents(draw, fk_series, row_count, at_least)
|
|
480
|
+
coverage[column.name] = cover
|
|
481
|
+
if cover.covered < cover.parents:
|
|
482
|
+
assert fk_target is not None
|
|
483
|
+
_uncovered_parents.append(
|
|
484
|
+
f"{table_name}: {row_count} rows cover {cover.covered} of {cover.parents} "
|
|
485
|
+
f"{fk_target[0]}; at_least {at_least} on {column.name} needs {cover.needed}"
|
|
486
|
+
)
|
|
487
|
+
return values
|
|
488
|
+
|
|
489
|
+
|
|
411
490
|
def _mirror_attributes(
|
|
412
491
|
df: pd.DataFrame,
|
|
413
492
|
table_name: str,
|
|
@@ -591,12 +670,16 @@ def _deduplicate_composite_keys(
|
|
|
591
670
|
as_of: AsOf = None,
|
|
592
671
|
time_profile: Optional[TimeProfile] = None,
|
|
593
672
|
skew: float = 0.0,
|
|
673
|
+
keep: Optional[Mapping[str, Any]] = None,
|
|
594
674
|
) -> pd.DataFrame:
|
|
595
675
|
"""
|
|
596
676
|
Regenerate colliding rows for any pk/unique composite key declared via an
|
|
597
677
|
`indexes {}` block, so the generated seed data respects that constraint.
|
|
598
678
|
Bounded retry mirrors generate.faker._deduplicate: give up gracefully on
|
|
599
679
|
a tiny value space rather than looping forever.
|
|
680
|
+
|
|
681
|
+
`keep` masks, per foreign-key column, the rows an `at_least` gave a parent:
|
|
682
|
+
such a row keeps it, and only the key's other columns are drawn again.
|
|
600
683
|
"""
|
|
601
684
|
columns_by_name = {column.name: column for column in table_def.columns}
|
|
602
685
|
|
|
@@ -638,11 +721,18 @@ def _deduplicate_composite_keys(
|
|
|
638
721
|
for position, idx in enumerate(df.index):
|
|
639
722
|
combo = combos[position]
|
|
640
723
|
attempts = 0
|
|
724
|
+
row_columns = regen_columns
|
|
725
|
+
if keep:
|
|
726
|
+
row_columns = [
|
|
727
|
+
(name, col) for name, col in regen_columns if not _kept(keep, name, position)
|
|
728
|
+
]
|
|
729
|
+
if not row_columns:
|
|
730
|
+
attempts = max_attempts
|
|
641
731
|
while combo in seen and attempts < max_attempts:
|
|
642
732
|
# Regenerate every column of the key (not just the last one) so
|
|
643
733
|
# the retry can actually reach unused combinations, not just
|
|
644
734
|
# unused values of a single column.
|
|
645
|
-
for col_name, col_def in
|
|
735
|
+
for col_name, col_def in row_columns:
|
|
646
736
|
if col_name in fk_pools:
|
|
647
737
|
df.at[idx, col_name] = random.choice(fk_pools[col_name])
|
|
648
738
|
else:
|
|
@@ -671,6 +761,11 @@ def _deduplicate_composite_keys(
|
|
|
671
761
|
return df
|
|
672
762
|
|
|
673
763
|
|
|
764
|
+
def _kept(keep: Mapping[str, Any], column: str, position: int) -> bool:
|
|
765
|
+
mask = keep.get(column)
|
|
766
|
+
return mask is not None and bool(mask[position])
|
|
767
|
+
|
|
768
|
+
|
|
674
769
|
def _resolve_self_referencing_fks(
|
|
675
770
|
df: pd.DataFrame,
|
|
676
771
|
table_def: TableDef,
|
|
@@ -680,6 +775,7 @@ def _resolve_self_referencing_fks(
|
|
|
680
775
|
as_of: AsOf = None,
|
|
681
776
|
time_profile: Optional[TimeProfile] = None,
|
|
682
777
|
skew: float = 0.0,
|
|
778
|
+
coverage: Optional[dict[str, Coverage]] = None,
|
|
683
779
|
) -> pd.DataFrame:
|
|
684
780
|
"""
|
|
685
781
|
Re-generate any FK column that references its own table (e.g. a
|
|
@@ -710,9 +806,8 @@ def _resolve_self_referencing_fks(
|
|
|
710
806
|
continue
|
|
711
807
|
|
|
712
808
|
ensure_unique = "pk" in column.settings or "unique" in column.settings
|
|
713
|
-
|
|
809
|
+
draw = _drawer(
|
|
714
810
|
column=column,
|
|
715
|
-
row_count=row_count,
|
|
716
811
|
fk_series=df[parent_column],
|
|
717
812
|
ensure_unique=ensure_unique,
|
|
718
813
|
force_not_null=column.name in composite_pk_columns,
|
|
@@ -721,6 +816,15 @@ def _resolve_self_referencing_fks(
|
|
|
721
816
|
time_profile=time_profile,
|
|
722
817
|
skew=skew,
|
|
723
818
|
)
|
|
819
|
+
df[column.name] = _covered(
|
|
820
|
+
draw,
|
|
821
|
+
column,
|
|
822
|
+
table_name,
|
|
823
|
+
fk_target,
|
|
824
|
+
df[parent_column],
|
|
825
|
+
row_count,
|
|
826
|
+
coverage if coverage is not None else {},
|
|
827
|
+
)
|
|
724
828
|
|
|
725
829
|
return df
|
|
726
830
|
|